diff --git a/.arcconfig b/.arcconfig --- a/.arcconfig +++ b/.arcconfig @@ -1,5 +1,5 @@ { - "phabricator.uri" : "https://phabricator.services.mozilla.com/", + "phabricator.uri" : "https://phabricator-dev.allizom.org/", "repository.callsign": "MOZILLACENTRAL", "history.immutable": false } diff --git a/media/libdav1d/moz.yaml b/media/libdav1d/moz.yaml --- a/media/libdav1d/moz.yaml +++ b/media/libdav1d/moz.yaml @@ -18,11 +18,11 @@ # Usually different from repository url url: https://code.videolan.org/videolan/dav1d # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: commit 39667c751d427e447cbe8be783cfecd296659e24 (2019-12-02T18:19:06.000+01:00). + release: commit 604d93c5f77d3e250a27d7d93570b72c5e0c2df5 (2020-04-07T16:21:53.000+02:00). # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libdav1d/vcs_version.h b/media/libdav1d/vcs_version.h --- a/media/libdav1d/vcs_version.h +++ b/media/libdav1d/vcs_version.h @@ -1,2 +1,2 @@ /* auto-generated, do not edit */ -#define DAV1D_VERSION "0.5.2-0-g39667c7" +#define DAV1D_VERSION "604d93c5f77d3e250a27d7d93570b72c5e0c2df5" diff --git a/third_party/dav1d/.gitlab-ci.yml b/third_party/dav1d/.gitlab-ci.yml --- a/third_party/dav1d/.gitlab-ci.yml +++ b/third_party/dav1d/.gitlab-ci.yml @@ -1,19 +1,65 @@ stages: - style - build - test -style-check: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: style +.debian-amd64-common: + image: registry.videolan.org/dav1d-debian-unstable:20200306210534 + stage: build + tags: + - docker + - amd64 + +.debian-llvm-mingw-common: + image: registry.videolan.org/vlc-debian-llvm-mingw:20190218133533 + stage: build + tags: + - docker + - amd64 + +.debian-aarch64-common: + image: registry.videolan.org/dav1d-debian-buster-aarch64:20200218203017 + stage: build + tags: + - docker + - aarch64 + +.debian-armv7-common: + image: registry.videolan.org/dav1d-debian-unstable-armv7:20190202101732 + stage: build tags: - - debian + - docker + - armv7 + +.debian-ppc64le-common: + image: registry.videolan.org/dav1d-debian-unstable-ppc64le:20190606105121 + stage: build + tags: + - docker + - ppc64le + +.ubuntu-common: + image: registry.videolan.org/dav1d-ubuntu-bionic:20200121182340 + stage: build + tags: + - docker - amd64 + +.android-common: + image: registry.videolan.org/vlc-debian-android:20200323093226 + stage: build + tags: + - docker + - amd64 + +style-check: + extends: .debian-amd64-common + stage: style script: - - git grep -I -n -P "\t|\r| $" -- . ':(exclude)*/compat/*' && exit 1 - - git grep -I -n -i "david" -- . ':(exclude)THANKS.md' ':(exclude).gitlab-ci.yml' && exit 1 + - git grep -I -n -P "\t|\r| $" -- . ':(exclude)*/compat/*' && echo "Trailing whitespace" && exit 1 + - git grep -I -n -i -e 'david' --and --not -e 'copyright' -- . ':(exclude)THANKS.md' ':(exclude).gitlab-ci.yml' && echo "Misspelled dav1d" && exit 1 - git grep -I -l -z "" -- . ':(exclude)*/compat/*' | while IFS= read -r -d '' i; do if [ -n "$(tail -c 1 "$i")" ]; then echo "No newline at end of $i"; exit 1; fi; @@ -33,58 +79,57 @@ exit 1; fi; done build-debian: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build + extends: .debian-amd64-common tags: - - debian + - docker + - avx2 - amd64 script: - - meson build --buildtype release --werror + - meson build --buildtype release + --werror - ninja -C build - cd build && meson test -v build-debian-static: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build - tags: - - debian - - amd64 + extends: .debian-amd64-common script: - - meson build --buildtype release --default-library static --werror + - meson build --buildtype release + --default-library static + --werror - ninja -C build - cd build && meson test -v - nm -A -g src/libdav1d.a | grep " [ABCDGRST] " | (! grep -v " _*dav1d_") build-debian32: - image: registry.videolan.org:5000/dav1d-debian-unstable:20181218135732 - stage: build - tags: - - debian - - amd64 + extends: .debian-amd64-common script: - meson build --buildtype release --werror - --cross-file /opt/crossfiles/linux32.meson + --cross-file package/crossfiles/i686-linux32.meson - ninja -C build - cd build && meson test -v +build-debian-examples: + extends: .debian-amd64-common + script: + - meson build --buildtype release + --werror + -Denable_examples=true + - ninja -C build + build-win32: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build - tags: - - debian - - amd64 + extends: .debian-amd64-common script: - wineserver -p && wine wineboot - meson build --buildtype release --werror --libdir lib --prefix "$(pwd)/build/dav1d_install" - --cross-file /opt/crossfiles/i686-w64-mingw32.meson + --cross-file package/crossfiles/i686-w64-mingw32.meson -Ddefault_library=both - ninja -C build - ninja -C build install - cd build && meson test -v - i686-w64-mingw32-nm -A -g src/libdav1d.a | grep " [ABCDGRST] " | (! grep -E -v " \.| _*dav1d_") @@ -93,37 +138,29 @@ paths: - build/dav1d_install/ expire_in: 1 week build-win32-unaligned-stack: - image: registry.videolan.org:5000/vlc-debian-llvm-mingw:20190218133533 - stage: build - tags: - - debian - - amd64 + extends: .debian-llvm-mingw-common script: - wineserver -p && wine wineboot - meson build --buildtype release --werror - --cross-file /opt/crossfiles/i686-w64-mingw32.meson + --cross-file package/crossfiles/i686-w64-mingw32.meson -Dstack_alignment=4 - ninja -C build - cd build && meson test -v build-win64: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build - tags: - - debian - - amd64 + extends: .debian-amd64-common script: - wineserver -p && wine wineboot - meson build --buildtype release --werror --libdir lib --prefix "$(pwd)/build/dav1d_install" - --cross-file /opt/crossfiles/x86_64-w64-mingw32.meson + --cross-file package/crossfiles/x86_64-w64-mingw32.meson -Ddefault_library=both - ninja -C build - ninja -C build install - cd build && meson test -v - x86_64-w64-mingw32-nm -A -g src/libdav1d.a | grep " [ABCDGRST] " | (! grep -E -v " \.| _*dav1d_") @@ -132,15 +169,11 @@ paths: - build/dav1d_install/ expire_in: 1 week build-win-arm32: - image: registry.videolan.org:5000/vlc-debian-llvm-mingw:20190218133533 - stage: build - tags: - - debian - - amd64 + extends: .debian-llvm-mingw-common script: - meson build --buildtype release --werror --libdir lib --prefix "$(pwd)/build/dav1d_install" @@ -148,15 +181,11 @@ -Ddefault_library=both - ninja -C build - armv7-w64-mingw32-nm -A -g build/src/libdav1d.a | grep " [ABCDGRST] " | (! grep -E -v " \.| _*dav1d_") build-win-arm64: - image: registry.videolan.org:5000/vlc-debian-llvm-mingw:20190218133533 - stage: build - tags: - - debian - - amd64 + extends: .debian-llvm-mingw-common script: - meson build --buildtype release --werror --libdir lib --prefix "$(pwd)/build/dav1d_install" @@ -169,79 +198,118 @@ name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" paths: - build/dav1d_install/ expire_in: 1 week +.build-android-common: + extends: .android-common + script: + - meson build --buildtype release + --werror + --libdir lib + --prefix "$(pwd)/build/dav1d_install" + --cross-file $CROSSFILE + -Ddefault_library=both + - ninja -C build + - ninja -C build install + +build-android-armv7: + extends: .build-android-common + variables: + CROSSFILE: package/crossfiles/arm-android.meson + except: + - tags + +build-android-aarch64: + extends: .build-android-common + variables: + CROSSFILE: package/crossfiles/aarch64-android.meson + except: + - tags + +build-android-armv7-release: + extends: build-android-armv7 + except: + only: + refs: + - tags@videolan/dav1d + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + paths: + - build/dav1d_install/ + expire_in: 1 week + +build-android-aarch64-release: + extends: build-android-aarch64 + except: + only: + refs: + - tags@videolan/dav1d + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + paths: + - build/dav1d_install/ + expire_in: 1 week + build-debian-aarch64: - stage: build - image: registry.videolan.org:5000/dav1d-debian-unstable-aarch64:20181122182457 - tags: - - aarch64 - - debian + extends: .debian-aarch64-common script: - - meson build --buildtype debugoptimized --werror + - meson build --buildtype debugoptimized + --werror - ninja -C build - cd build && meson test -v build-debian-aarch64-clang-5: - stage: build - image: registry.videolan.org:5000/dav1d-debian-unstable-aarch64:20181122182457 - tags: - - aarch64 - - debian + extends: .debian-aarch64-common + variables: + CC: clang-5.0 + CFLAGS: '-integrated-as' script: - - env CC=clang-5.0 CFLAGS='-integrated-as' meson build --buildtype release + - meson build --buildtype release - ninja -C build - cd build && meson test -v build-macos: stage: build tags: - macos script: - - meson build --buildtype release -Ddefault_library=both --werror + - meson build --buildtype release + -Ddefault_library=both + --werror - ninja -C build - cd build && meson test -v build-debian-werror: - image: registry.videolan.org:5000/dav1d-debian-unstable-aarch64:20181122182457 - stage: build - tags: - - aarch64 - - debian + extends: .debian-aarch64-common + variables: + CC: clang-7 script: - - env CC='clang-7' meson build --buildtype debug --werror + - meson build --buildtype debug + --werror - ninja -C build build-debian-armv7: - stage: build - image: registry.videolan.org:5000/dav1d-debian-unstable-armv7:20190202101732 - tags: - - armv7 - - debian + extends: .debian-armv7-common script: - - meson build --buildtype debugoptimized --werror + - linux32 meson build --buildtype debugoptimized + --werror - ninja -C build - cd build && meson test -v build-debian-armv7-clang-5: - stage: build - image: registry.videolan.org:5000/dav1d-debian-unstable-armv7:20190202101732 - tags: - - armv7 - - debian + extends: .debian-armv7-common + variables: + CC: clang-5.0 + CFLAGS: '-integrated-as' script: - - env CC=clang-5.0 CFLAGS='-integrated-as' meson build --buildtype release + - linux32 meson build --buildtype release - ninja -C build - cd build && meson test -v build-ubuntu-snap: - stage: build - image: registry.videolan.org:5000/dav1d-ubuntu-bionic:20190221154127 - tags: - - debian - - amd64 + extends: .ubuntu-common script: - cd package/snap && snapcraft snap - | if [ "$CI_PROJECT_NAMESPACE" = "videolan" ]; then echo $SNAP_LOGIN | base64 --decode | snapcraft login --with - @@ -254,219 +322,187 @@ - package/snap/dav1d_*.snap expire_in: 1 week allow_failure: true build-debian-ppc64le: - stage: build - image: registry.videolan.org/dav1d-debian-unstable-ppc64le:20190606105121 - tags: - - ppc64le - - docker + extends: .debian-ppc64le-common script: - - meson build --buildtype release --werror + - meson build --buildtype release + --werror - ninja -C build - cd build && meson test -v -test-debian: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 +.test-common: stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 cache: key: testdata.git-20190215 paths: - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype release -Dtestdata_tests=true -Dlogging=false - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-unaligned-stack: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype release -Dtestdata_tests=true -Dlogging=false -Dstack_alignment=16 - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-asan: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - variables: - ASAN_OPTIONS: 'detect_leaks=0' - script: + before_script: - test -d cache || mkdir cache - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype debugoptimized -Dtestdata_tests=true -Dlogging=false -Db_sanitize=address -Denable_asm=false - - ninja -C build - - cd build && time meson test -v --setup=sanitizer - dependencies: [] - -test-debian-msan: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - variables: - MSAN_OPTIONS: 'exitcode=1' - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - env CC=clang meson build --buildtype debugoptimized -Dtestdata_tests=true -Dlogging=false -Db_sanitize=memory -Db_lundef=false -Denable_asm=false - - ninja -C build - - cd build && time meson test -v --setup=sanitizer dependencies: [] -test-debian-ubsan: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test +test-debian: + extends: + - .debian-amd64-common + - .test-common needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - variables: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - env CC=clang meson build --buildtype debugoptimized -Dtestdata_tests=true -Dlogging=false -Db_sanitize=undefined -Db_lundef=false -Denable_asm=false - - ninja -C build - - cd build && time meson test -v --setup=sanitizer - dependencies: [] - -test-win64: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-win64"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - wineserver -p && wine wineboot - - meson build --buildtype release - -Dtestdata_tests=true - -Dlogging=false - --cross-file /opt/crossfiles/x86_64-w64-mingw32.meson - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-aarch64: - image: registry.videolan.org:5000/dav1d-debian-unstable-aarch64:20181122182457 - stage: test - needs: ["build-debian-aarch64"] - tags: - - aarch64 - - debian - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - meson build --buildtype release -Dtestdata_tests=true -Dlogging=false - ninja -C build - cd build && time meson test -v - dependencies: [] -test-debian-ppc64le: - image: registry.videolan.org/dav1d-debian-unstable-ppc64le:20190606105121 - stage: test - needs: ["build-debian-ppc64le"] +test-debian-unaligned-stack: + extends: + - .debian-amd64-common + - .test-common + needs: ["build-debian"] tags: - - ppc64le - docker - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ + - avx2 + - amd64 + script: + - meson build --buildtype release + -Dtestdata_tests=true + -Dlogging=false + -Dstack_alignment=16 + - ninja -C build + - cd build && time meson test -v + +test-debian-asan: + extends: + - .debian-amd64-common + - .test-common + needs: ["build-debian"] + variables: + ASAN_OPTIONS: 'detect_leaks=0' + script: + - meson build --buildtype debugoptimized + -Dtestdata_tests=true + -Dlogging=false + -Db_sanitize=address + -Denable_asm=false + - ninja -C build + - cd build && time meson test -v --setup=sanitizer + +test-debian-msan: + extends: + - .debian-amd64-common + - .test-common + needs: ["build-debian"] + variables: + MSAN_OPTIONS: 'exitcode=1' + CC: clang script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data + - meson build --buildtype debugoptimized + -Dtestdata_tests=true + -Dlogging=false + -Db_sanitize=memory + -Db_lundef=false + -Denable_asm=false + - ninja -C build + - cd build && time meson test -v --setup=sanitizer + +test-debian-ubsan: + extends: + - .debian-amd64-common + - .test-common + needs: ["build-debian"] + variables: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' + CC: clang + script: + - meson build --buildtype debugoptimized + -Dtestdata_tests=true + -Dlogging=false + -Db_sanitize=undefined + -Db_lundef=false + -Denable_asm=false + - ninja -C build + - cd build && time meson test -v --setup=sanitizer + +test-win64: + extends: + - .debian-amd64-common + - .test-common + needs: ["build-win64"] + tags: + - docker + - avx2 + - amd64 + script: + - wineserver -p && wine wineboot + - meson build --buildtype release + -Dtestdata_tests=true + -Dlogging=false + --cross-file package/crossfiles/x86_64-w64-mingw32.meson + - ninja -C build + - cd build && time meson test -v + +test-debian-aarch64: + extends: + - .debian-aarch64-common + - .test-common + needs: ["build-debian-aarch64"] + script: - meson build --buildtype release -Dtestdata_tests=true -Dlogging=false - ninja -C build - cd build && time meson test -v - dependencies: [] + +test-debian-ppc64le: + extends: + - .debian-ppc64le-common + - .test-common + needs: ["build-debian-ppc64le"] + script: + - meson build --buildtype release + -Dtestdata_tests=true + -Dlogging=false + - ninja -C build + - cd build && time meson test -v test-debian-armv7-clang-5: - stage: test - image: registry.videolan.org:5000/dav1d-debian-unstable-armv7:20190202101732 + extends: + - .debian-armv7-common + - .test-common needs: ["build-debian-armv7-clang-5"] - tags: - - armv7 - - debian - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ + variables: + CC: clang-5.0 + CFLAGS: '-integrated-as' script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - env CC=clang-5.0 CFLAGS='-integrated-as' meson build --buildtype release - -Dtestdata_tests=true - -Dlogging=false + - linux32 meson build --buildtype release + -Dtestdata_tests=true + -Dlogging=false - ninja -C build - cd build && time meson test -v - dependencies: [] + +.pages-common: + extends: .debian-amd64-common + script: + - meson build --buildtype release + --werror + - ninja -C build doc/html + - mv build/doc/html public + artifacts: + paths: + - public + +build-pages: + extends: .pages-common + except: + refs: + - master + +pages: + extends: .pages-common + only: + refs: + - master + changes: + - include/dav1d/* diff --git a/third_party/dav1d/NEWS b/third_party/dav1d/NEWS --- a/third_party/dav1d/NEWS +++ b/third_party/dav1d/NEWS @@ -1,5 +1,28 @@ +Changes for 0.6.0 'Gyrfalcon': +------------------------------ + +0.6.0 is a major release for dav1d: + - New ARM64 optimizations for the 10/12bit depth: + - mc_avg, mc_w_avg, mc_mask + - mc_put/mc_prep 8tap/bilin + - mc_warp_8x8 + - mc_w_mask + - mc_blend + - wiener + - SGR + - loopfilter + - cdef + - New AVX-512 optimizations for prep_bilin, prep_8tap, cdef_filter, mc_avg/w_avg/mask + - New SSSE3 optimizations for film grain + - New AVX2 optimizations for msac_adapt16 + - Fix rare mismatches against the reference decoder, notably because of clipping + - Improvements on ARM64 on msac, cdef and looprestoration optimizations + - Improvements on AVX2 optimizations for cdef_filter + - Improvements in the C version for itxfm, cdef_filter + + Changes for 0.5.2 'Asiatic Cheetah': ------------------------------------ 0.5.2 is a small release improving speed for ARM32 and adding minor features: - ARM32 optimizations for loopfilter, ipred_dc|h|v @@ -30,11 +53,11 @@ - Speed improvements on decode_coef and MSAC - NEON optimizations for blend*, w_mask_, ipred functions for ARM64 - NEON optimizations for CDEF and warp on ARM32 - SSE2 optimizations for MSAC hi_tok decoding - SSSE3 optimizations for deblocking loopfilters and warp_affine - - AVX-2 optimizations for film grain and ipred_z2 + - AVX2 optimizations for film grain and ipred_z2 - SSE4 optimizations for warp_affine - VSX optimizations for wiener - Fix inverse transform overflows in x86 and NEON asm - Fix integer overflows with large frames - Improve film grain generation to match reference code @@ -79,11 +102,11 @@ Changes for 0.2.2 (0.3.0-rc) 'Antelope': ----------------------------- - Large improvement on MSAC decoding with SSE, bringing 4-6% speed increase - The impact is important on SSSE3, SSE4 and AVX-2 cpus + The impact is important on SSSE3, SSE4 and AVX2 cpus - SSSE3 optimizations for all blocks size in itx - SSSE3 optimizations for ipred_paeth and ipred_cfl (420, 422 and 444) - Speed improvements on CDEF for SSE4 CPUs - NEON optimizations for SGR and loop filter - Minor crashes, improvements and build changes @@ -91,21 +114,21 @@ Changes for 0.2.1 'Antelope': ---------------------------- - SSSE3 optimization for cdef_dir - - AVX-2 improvements of the existing CDEF optimizations + - AVX2 improvements of the existing CDEF optimizations - NEON improvements of the existing CDEF and wiener optimizations - Clarification about the numbering/versionning scheme Changes for 0.2.0 'Antelope': ---------------------------- - ARM64 and ARM optimizations using NEON instructions - SSSE3 optimizations for both 32 and 64bits - - More AVX-2 assembly, reaching almost completion + - More AVX2 assembly, reaching almost completion - Fix installation of includes - Rewrite inverse transforms to avoid overflows - Snap packaging for Linux - Updated API (ABI and API break) - Fixes for un-decodable samples @@ -116,8 +139,8 @@ Initial release of dav1d, the fast and small AV1 decoder. - Support for all features of the AV1 bitstream - Support for all bitdepth, 8, 10 and 12bits - Support for all chroma subsamplings 4:2:0, 4:2:2, 4:4:4 *and* grayscale - - Full acceleration for AVX-2 64bits processors, making it the fastest decoder + - Full acceleration for AVX2 64bits processors, making it the fastest decoder - Partial acceleration for SSSE3 processors - Partial acceleration for NEON processors diff --git a/third_party/dav1d/README.md b/third_party/dav1d/README.md --- a/third_party/dav1d/README.md +++ b/third_party/dav1d/README.md @@ -22,11 +22,11 @@ The reasoning behind this decision is the same as for libvorbis, see [RMS on vorbis](https://lwn.net/2001/0301/a/rms-ov-license.php3). # Roadmap -The plan is the folllowing: +The plan is the following: ### Reached 1. Complete C implementation of the decoder, 2. Provide a usable API, 3. Port to most platforms, @@ -71,15 +71,37 @@ The [VideoLAN Code of Conduct](https://wiki.videolan.org/CoC) applies to this project. # Compile -1. Install [Meson](https://mesonbuild.com/) (0.47 or higher), [Ninja](https://ninja-build.org/), and, for x86\* targets, [nasm](https://nasm.us/) (2.13.02 or higher) +1. Install [Meson](https://mesonbuild.com/) (0.47 or higher), [Ninja](https://ninja-build.org/), and, for x86\* targets, [nasm](https://nasm.us/) (2.14 or higher) 2. Run `mkdir build && cd build` to create a build directory and enter it 3. Run `meson ..` to configure meson, add `--default-library=static` if static linking is desired 4. Run `ninja` to compile +## Cross-Compilation for 32- or 64-bit Windows, 32-bit Linux + +If you're on a linux build machine trying to compile .exe for a Windows target/host machine, run + +``` +meson build --cross-file=package/crossfiles/x86_64-w64-mingw32.meson +``` + +or, for 32-bit: + +``` +meson build --cross-file=package/crossfiles/i686-w64-mingw32.meson +``` + +`mingw-w64` is a pre-requisite and should be installed on your linux machine via your preferred method or package manager. Note the binary name formats may differ between distributions. Verify the names, and use `alias` if certain binaries cannot be found. + +For 32-bit linux, run + +``` +meson build --cross-file=package/crossfiles/i686-linux32.meson +``` + # Run tests 1. In the root directory, run `git clone https://code.videolan.org/videolan/dav1d-test-data.git tests/dav1d-test-data` to fetch the test data repository 2. During meson configuration, specify `-Dtestdata_tests=true` 3. Run `meson test -v` after compiling diff --git a/third_party/dav1d/doc/meson.build b/third_party/dav1d/doc/meson.build --- a/third_party/dav1d/doc/meson.build +++ b/third_party/dav1d/doc/meson.build @@ -21,12 +21,13 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. doxygen = find_program('doxygen', required: false) +dot = find_program('dot', required: false) -if doxygen.found() +if doxygen.found() and dot.found() conf_data = configuration_data() conf_data.set('DOXYGEN_INPUT', join_paths(meson.source_root(), 'include/dav1d')) conf_data.set('DOXYGEN_STRIP', join_paths(meson.source_root(), 'include')) conf_data.set('DOXYGEN_OUTPUT', meson.current_build_dir()) doxyfile = configure_file(input: 'Doxyfile.in', diff --git a/third_party/dav1d/examples/dav1dplay.c b/third_party/dav1d/examples/dav1dplay.c --- a/third_party/dav1d/examples/dav1dplay.c +++ b/third_party/dav1d/examples/dav1dplay.c @@ -1024,13 +1024,10 @@ // Store current ticks for stats calculation uint32_t decoder_start = SDL_GetTicks(); Dav1dPlaySettings settings = rd_ctx->settings; - // Init dav1d tools demuxers - init_demuxers(); - if ((res = input_open(&in_ctx, "ivf", settings.inputfile, fps, &total, timebase)) < 0) { fprintf(stderr, "Failed to open demuxer\n"); diff --git a/third_party/dav1d/examples/meson.build b/third_party/dav1d/examples/meson.build --- a/third_party/dav1d/examples/meson.build +++ b/third_party/dav1d/examples/meson.build @@ -35,11 +35,11 @@ # dav1d player sources dav1dplay_sources = files( 'dav1dplay.c', ) -sdl2_dependency = dependency('sdl2', version: '>= 2.0.1', required: false) +sdl2_dependency = dependency('sdl2', version: '>= 2.0.1', required: true) if sdl2_dependency.found() placebo_dependency = dependency('libplacebo', version: '>= 1.18.0', required: false) vulkan_dependency = dependency('vulkan', required: false) sdl_has_vulkan = cc.has_header('SDL_vulkan.h', dependencies: [sdl2_dependency]) diff --git a/third_party/dav1d/include/common/attributes.h b/third_party/dav1d/include/common/attributes.h --- a/third_party/dav1d/include/common/attributes.h +++ b/third_party/dav1d/include/common/attributes.h @@ -41,19 +41,22 @@ #define ATTR_FORMAT_PRINTF(fmt, attr) #define COLD #endif #if ARCH_X86_64 -/* x86-64 needs 32-byte alignment for AVX2. */ +/* x86-64 needs 32- and 64-byte alignment for AVX2 and AVX-512. */ +#define ALIGN_64_VAL 64 #define ALIGN_32_VAL 32 #define ALIGN_16_VAL 16 #elif ARCH_X86_32 || ARCH_ARM || ARCH_AARCH64 || ARCH_PPC64LE /* ARM doesn't benefit from anything more than 16-byte alignment. */ +#define ALIGN_64_VAL 16 #define ALIGN_32_VAL 16 #define ALIGN_16_VAL 16 #else /* No need for extra alignment on platforms without assembly. */ +#define ALIGN_64_VAL 8 #define ALIGN_32_VAL 8 #define ALIGN_16_VAL 8 #endif /* @@ -74,13 +77,14 @@ * API for stack alignment (ALIGN_STK_$align()) of variables like: * uint8_t var[1][2][3][4] * becomes: * ALIGN_STK_$align(uint8_t, var, 1, [2][3][4]) */ +#define ALIGN_STK_64(type, var, sz1d, sznd) \ + ALIGN(type var[sz1d]sznd, ALIGN_64_VAL) #define ALIGN_STK_32(type, var, sz1d, sznd) \ ALIGN(type var[sz1d]sznd, ALIGN_32_VAL) -// as long as stack is itself 16-byte aligned, this works (win64, gcc) #define ALIGN_STK_16(type, var, sz1d, sznd) \ ALIGN(type var[sz1d]sznd, ALIGN_16_VAL) /* * Forbid inlining of a function: @@ -90,10 +94,16 @@ #define NOINLINE __declspec(noinline) #else /* !_MSC_VER */ #define NOINLINE __attribute__((noinline)) #endif /* !_MSC_VER */ +#ifdef __clang__ +#define NO_SANITIZE(x) __attribute__((no_sanitize(x))) +#else +#define NO_SANITIZE(x) +#endif + #if defined(NDEBUG) && (defined(__GNUC__) || defined(__clang__)) #define assert(x) do { if (!(x)) __builtin_unreachable(); } while (0) #elif defined(NDEBUG) && defined(_MSC_VER) #define assert __assume #else diff --git a/third_party/dav1d/include/common/bitdepth.h b/third_party/dav1d/include/common/bitdepth.h --- a/third_party/dav1d/include/common/bitdepth.h +++ b/third_party/dav1d/include/common/bitdepth.h @@ -29,10 +29,12 @@ #define DAV1D_COMMON_BITDEPTH_H #include #include +#include "common/attributes.h" + #if !defined(BITDEPTH) typedef void pixel; typedef void coef; #define HIGHBD_DECL_SUFFIX /* nothing */ #define HIGHBD_CALL_SUFFIX /* nothing */ @@ -45,16 +47,18 @@ #define pixel_copy memcpy #define pixel_set memset #define iclip_pixel iclip_u8 #define PIX_HEX_FMT "%02x" #define bitfn(x) x##_8bpc -#define PXSTRIDE(x) x +#define BF(x, suffix) x##_8bpc_##suffix +#define PXSTRIDE(x) (x) #define highbd_only(x) #define HIGHBD_DECL_SUFFIX /* nothing */ #define HIGHBD_CALL_SUFFIX /* nothing */ #define HIGHBD_TAIL_SUFFIX /* nothing */ #define bitdepth_from_max(x) 8 +#define BITDEPTH_MAX 0xff #elif BITDEPTH == 16 typedef uint16_t pixel; typedef int32_t coef; #define PIXEL_TYPE uint16_t #define COEF_TYPE int32_t @@ -67,12 +71,17 @@ #define iclip_pixel(x) iclip(x, 0, bitdepth_max) #define HIGHBD_DECL_SUFFIX , const int bitdepth_max #define HIGHBD_CALL_SUFFIX , f->bitdepth_max #define HIGHBD_TAIL_SUFFIX , bitdepth_max #define bitdepth_from_max(bitdepth_max) (32 - clz(bitdepth_max)) +#define BITDEPTH_MAX bitdepth_max #define bitfn(x) x##_16bpc -#define PXSTRIDE(x) (x >> 1) +#define BF(x, suffix) x##_16bpc_##suffix +static inline ptrdiff_t PXSTRIDE(const ptrdiff_t x) { + assert(!(x & 1)); + return x >> 1; +} #define highbd_only(x) x #else #error invalid value for bitdepth #endif #define bytefn(x) bitfn(x) diff --git a/third_party/dav1d/include/common/mem.h b/third_party/dav1d/include/common/mem.h --- a/third_party/dav1d/include/common/mem.h +++ b/third_party/dav1d/include/common/mem.h @@ -35,17 +35,17 @@ #endif #include "common/attributes.h" /* - * Allocate 32-byte aligned memory. The return value can be released - * by calling the standard free() function. + * Allocate align-byte aligned memory. The return value can be released + * by calling the dav1d_free_aligned() function. */ static inline void *dav1d_alloc_aligned(size_t sz, size_t align) { + assert(!(align & (align - 1))); #ifdef HAVE_POSIX_MEMALIGN void *ptr; - assert(!(align & (align - 1))); if (posix_memalign(&ptr, align, sz)) return NULL; return ptr; #elif defined(HAVE_ALIGNED_MALLOC) return _aligned_malloc(sz, align); #elif defined(HAVE_MEMALIGN) diff --git a/third_party/dav1d/include/dav1d/dav1d.h b/third_party/dav1d/include/dav1d/dav1d.h --- a/third_party/dav1d/include/dav1d/dav1d.h +++ b/third_party/dav1d/include/dav1d/dav1d.h @@ -47,11 +47,11 @@ #define DAV1D_MAX_TILE_THREADS 64 typedef struct Dav1dLogger { void *cookie; ///< Custom data to pass to the callback. /** - * Logger callback. Default prints to stderr. May be NULL to disable logging. + * Logger callback. May be NULL to disable logging. * * @param cookie Custom pointer passed to all calls. * @param format The vprintf compatible format string. * @param ap List of arguments referenced by the format string. */ @@ -64,12 +64,12 @@ int apply_grain; int operating_point; ///< select an operating point for scalable AV1 bitstreams (0 - 31) int all_layers; ///< output all spatial layers of a scalable AV1 biststream unsigned frame_size_limit; ///< maximum frame size, in pixels (0 = unlimited) uint8_t reserved[32]; ///< reserved for future use - Dav1dPicAllocator allocator; - Dav1dLogger logger; + Dav1dPicAllocator allocator; ///< Picture allocator callback. + Dav1dLogger logger; ///< Logger callback. } Dav1dSettings; /** * Get library version. */ diff --git a/third_party/dav1d/include/dav1d/headers.h b/third_party/dav1d/include/dav1d/headers.h --- a/third_party/dav1d/include/dav1d/headers.h +++ b/third_party/dav1d/include/dav1d/headers.h @@ -316,29 +316,30 @@ int num_uv_points[2]; uint8_t uv_points[2][10][2 /* value, scaling */]; int scaling_shift; int ar_coeff_lag; int8_t ar_coeffs_y[24]; - int8_t ar_coeffs_uv[2][25]; - int ar_coeff_shift; + int8_t ar_coeffs_uv[2][25 + 3 /* padding for alignment purposes */]; + uint64_t ar_coeff_shift; int grain_scale_shift; int uv_mult[2]; int uv_luma_mult[2]; int uv_offset[2]; int overlap_flag; int clip_to_restricted_range; } Dav1dFilmGrainData; typedef struct Dav1dFrameHeader { + struct { + Dav1dFilmGrainData data; + int present, update; + } film_grain; ///< film grain parameters enum Dav1dFrameType frame_type; ///< type of the picture int width[2 /* { coded_width, superresolution_upscaled_width } */], height; int frame_offset; ///< frame number - struct { - int present, update; - Dav1dFilmGrainData data; - } film_grain; ///< film grain parameters - int temporal_id, spatial_id; ///< spatial and temporal id of the frame for SVC + int temporal_id; ///< temporal id of the frame for SVC + int spatial_id; ///< spatial id of the frame for SVC int show_existing_frame; int existing_frame_idx; int frame_id; int frame_presentation_delay; diff --git a/third_party/dav1d/include/dav1d/picture.h b/third_party/dav1d/include/dav1d/picture.h --- a/third_party/dav1d/include/dav1d/picture.h +++ b/third_party/dav1d/include/dav1d/picture.h @@ -82,12 +82,15 @@ */ Dav1dITUTT35 *itut_t35; uintptr_t reserved[4]; ///< reserved for future use - struct Dav1dRef *frame_hdr_ref, *seq_hdr_ref; ///< Frame parameter allocation origins - struct Dav1dRef *content_light_ref, *mastering_display_ref, *itut_t35_ref; ///< Metadata allocation origins + struct Dav1dRef *frame_hdr_ref; ///< Dav1dFrameHeader allocation origin + struct Dav1dRef *seq_hdr_ref; ///< Dav1dSequenceHeader allocation origin + struct Dav1dRef *content_light_ref; ///< Dav1dContentLightLevel allocation origin + struct Dav1dRef *mastering_display_ref; ///< Dav1dMasteringDisplay allocation origin + struct Dav1dRef *itut_t35_ref; ///< Dav1dITUTT35 allocation origin uintptr_t reserved_ref[4]; ///< reserved for future use struct Dav1dRef *ref; ///< Frame data allocation origin void *allocator_data; ///< pointer managed by the allocator } Dav1dPicture; diff --git a/third_party/dav1d/meson.build b/third_party/dav1d/meson.build --- a/third_party/dav1d/meson.build +++ b/third_party/dav1d/meson.build @@ -21,18 +21,18 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project('dav1d', ['c'], - version: '0.5.2', + version: '0.6.0', default_options: ['c_std=c99', 'warning_level=2', 'buildtype=release', 'b_ndebug=if-release'], meson_version: '>= 0.47.0') -dav1d_soname_version = '3.1.0' +dav1d_soname_version = '4.0.0' dav1d_api_version_array = dav1d_soname_version.split('.') dav1d_api_version_major = dav1d_api_version_array[0] dav1d_api_version_minor = dav1d_api_version_array[1] dav1d_api_version_revision = dav1d_api_version_array[2] @@ -82,17 +82,19 @@ # Arguments in test_args will be used even on feature tests test_args = [] optional_arguments = [] -# Define _POSIX_C_SOURCE to POSIX.1–2001 (IEEE Std 1003.1-2001) -test_args += '-D_POSIX_C_SOURCE=200112L' -add_project_arguments('-D_POSIX_C_SOURCE=200112L', language: 'c') - -if host_machine.system() == 'darwin' +if host_machine.system() == 'linux' + test_args += '-D_GNU_SOURCE' + add_project_arguments('-D_GNU_SOURCE', language: 'c') +elif host_machine.system() == 'darwin' test_args += '-D_DARWIN_C_SOURCE' add_project_arguments('-D_DARWIN_C_SOURCE', language: 'c') +else + test_args += '-D_POSIX_C_SOURCE=200112L' + add_project_arguments('-D_POSIX_C_SOURCE=200112L', language: 'c') endif if host_machine.system() == 'windows' cdata.set('_WIN32_WINNT', '0x0601') cdata.set('UNICODE', 1) # Define to 1 for Unicode (Wide Chars) APIs @@ -129,10 +131,34 @@ endif cdata.set('HAVE_CLOCK_GETTIME', 1) endif endif +# check for fseeko on android. It is not always available if _FILE_OFFSET_BITS is defined to 64 +have_fseeko = true +if host_machine.system() == 'android' + if not cc.has_function('fseeko', prefix : '#include ', args : test_args) + if cc.has_function('fseeko', prefix : '#include ', args : test_args + ['-U_FILE_OFFSET_BITS']) + warning('Files larger than 2 gigabytes might not be supported in the dav1d CLI tool.') + add_project_arguments('-U_FILE_OFFSET_BITS', language: 'c') + elif get_option('enable_tools') + error('dav1d CLI tool needs fseeko()') + else + have_fseeko = false + endif + endif +endif + +libdl_dependency = [] +if host_machine.system() == 'linux' + libdl_dependency = cc.find_library('dl', required : false) + if cc.has_function('dlsym', prefix : '#include ', args : test_args, dependencies : libdl_dependency) + cdata.set('HAVE_DLSYM', 1) + endif +endif + + # Header checks stdatomic_dependency = [] if not cc.check_header('stdatomic.h') if cc.get_id() == 'msvc' @@ -255,16 +281,16 @@ if host_machine.cpu_family().startswith('x86') if get_option('stack_alignment') > 0 stack_alignment = get_option('stack_alignment') elif host_machine.cpu_family() == 'x86_64' - if cc.has_argument('-mpreferred-stack-boundary=5') - stackalign_flag = ['-mpreferred-stack-boundary=5'] + if cc.has_argument('-mpreferred-stack-boundary=6') + stackalign_flag = ['-mpreferred-stack-boundary=6'] stackrealign_flag = ['-mincoming-stack-boundary=4'] stack_alignment = 32 - elif cc.has_argument('-mstack-alignment=32') - stackalign_flag = ['-mstack-alignment=32'] + elif cc.has_argument('-mstack-alignment=64') + stackalign_flag = ['-mstack-alignment=64'] stackrealign_flag = ['-mstackrealign'] stack_alignment = 32 else stack_alignment = 16 endif @@ -334,24 +360,15 @@ if cc.symbols_have_underscore_prefix() cdata.set10('PREFIX', true) cdata_asm.set10('PREFIX', true) endif -# Generate config.h -config_h_target = configure_file(output: 'config.h', configuration: cdata) - - - # # ASM specific stuff # if is_asm_enabled and host_machine.cpu_family().startswith('x86') - # Generate config.asm - config_asm_target = configure_file(output: 'config.asm', output_format: 'nasm', configuration: cdata_asm) - - # NASM compiler support nasm = find_program('nasm') # check NASM version @@ -364,16 +381,24 @@ out = nasm_r.stdout().strip().split() if out[1].to_lower() == 'version' if out[2].version_compare('<2.13.02') error('nasm 2.13.02 or later is required, found nasm @0@'.format(out[2])) + elif out[2].version_compare('<2.14') and get_option('enable_avx512') + error('nasm 2.14 or later is required for AVX-512 asm.\n' + + 'AVX-512 asm can be disabled with \'-Denable_avx512=false\'') endif + cdata.set10('HAVE_AVX512ICL', get_option('enable_avx512')) + cdata_asm.set10('HAVE_AVX512ICL', get_option('enable_avx512')) else error('unexpected nasm version string: @0@'.format(nasm_r.stdout())) endif endif + # Generate config.asm + config_asm_target = configure_file(output: 'config.asm', output_format: 'nasm', configuration: cdata_asm) + if host_machine.system() == 'windows' nasm_format = 'win' elif host_machine.system() == 'darwin' nasm_format = 'macho' else @@ -388,20 +413,24 @@ nasm_gen = generator(nasm, output: '@BASENAME@.obj', depfile: '@BASENAME@.obj.ndep', arguments: [ '-f', nasm_format, - '-I', '@SOURCE_DIR@/src/', + '-I', '@0@/src/'.format(meson.current_source_dir()), '-I', '@0@/'.format(meson.current_build_dir()), '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', '@EXTRA_ARGS@', '@INPUT@', '-o', '@OUTPUT@' ]) endif +# Generate config.h +config_h_target = configure_file(output: 'config.h', configuration: cdata) + + # # Include subdir meson.build files # The order is important! diff --git a/third_party/dav1d/meson_options.txt b/third_party/dav1d/meson_options.txt --- a/third_party/dav1d/meson_options.txt +++ b/third_party/dav1d/meson_options.txt @@ -8,10 +8,15 @@ option('enable_asm', type: 'boolean', value: true, description: 'Build asm files, if available') +option('enable_avx512', + type: 'boolean', + value: true, + description: 'Build AVX-512 asm files, requires nasm 2.14') + option('enable_tools', type: 'boolean', value: true, description: 'Build dav1d cli tools') diff --git a/third_party/dav1d/package/crossfiles/aarch64-android.meson b/third_party/dav1d/package/crossfiles/aarch64-android.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/aarch64-android.meson @@ -0,0 +1,16 @@ +[binaries] +c = 'aarch64-linux-android21-clang' +cpp = 'aarch64-linux-android21-clang++' +ar = 'aarch64-linux-android-ar' +strip = 'aarch64-linux-android-strip' +pkgconfig = 'pkg-config' +windres = 'aarch64-linux-android-windres' + +[properties] +needs_exe_wrapper = true + +[host_machine] +system = 'android' +cpu_family = 'aarch64' +endian = 'little' +cpu = 'aarch64' diff --git a/third_party/dav1d/package/crossfiles/arm-android.meson b/third_party/dav1d/package/crossfiles/arm-android.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/arm-android.meson @@ -0,0 +1,16 @@ +[binaries] +c = 'armv7a-linux-androideabi16-clang' +cpp = 'armv7a-linux-androideabi16-clang++' +ar = 'arm-linux-androideabi-ar' +strip = 'arm-linux-androideabi-strip' +pkgconfig = 'pkg-config' +windres = 'arm-linux-androideabi-windres' + +[properties] +needs_exe_wrapper = true + +[host_machine] +system = 'android' +cpu_family = 'arm' +endian = 'little' +cpu = 'arm' diff --git a/third_party/dav1d/package/crossfiles/i686-linux32.meson b/third_party/dav1d/package/crossfiles/i686-linux32.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/i686-linux32.meson @@ -0,0 +1,15 @@ +[binaries] +c = 'gcc' +cpp = 'g++' +ar = 'ar' +strip = 'strip' + +[properties] +c_link_args = ['-m32'] +c_args = ['-m32'] + +[host_machine] +system = 'linux' +cpu_family = 'x86' +cpu = 'i686' +endian = 'little' diff --git a/third_party/dav1d/package/crossfiles/i686-w64-mingw32.meson b/third_party/dav1d/package/crossfiles/i686-w64-mingw32.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/i686-w64-mingw32.meson @@ -0,0 +1,17 @@ +[binaries] +c = 'i686-w64-mingw32-gcc' +cpp = 'i686-w64-mingw32-g++' +ar = 'i686-w64-mingw32-ar' +strip = 'i686-w64-mingw32-strip' +windres = 'i686-w64-mingw32-windres' +exe_wrapper = ['wine'] + +[properties] +c_link_args = ['-static-libgcc'] +needs_exe_wrapper = true + +[host_machine] +system = 'windows' +cpu_family = 'x86' +cpu = 'i686' +endian = 'little' diff --git a/third_party/dav1d/package/crossfiles/x86_64-w64-mingw32.meson b/third_party/dav1d/package/crossfiles/x86_64-w64-mingw32.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/x86_64-w64-mingw32.meson @@ -0,0 +1,17 @@ +[binaries] +c = 'x86_64-w64-mingw32-gcc' +cpp = 'x86_64-w64-mingw32-g++' +ar = 'x86_64-w64-mingw32-ar' +strip = 'x86_64-w64-mingw32-strip' +windres = 'x86_64-w64-mingw32-windres' +exe_wrapper = ['wine'] + +[properties] +c_link_args = ['-static-libgcc'] +needs_exe_wrapper = true + +[host_machine] +system = 'windows' +cpu_family = 'x86_64' +cpu = 'x86_64' +endian = 'little' diff --git a/third_party/dav1d/src/arm/32/cdef.S b/third_party/dav1d/src/arm/32/cdef.S --- a/third_party/dav1d/src/arm/32/cdef.S +++ b/third_party/dav1d/src/arm/32/cdef.S @@ -146,24 +146,26 @@ .else vld1.8 {\dst\()}, [\src, :64], \incr .endif .endm -// void dav1d_cdef_paddingX_neon(uint16_t *tmp, const pixel *src, -// ptrdiff_t src_stride, const pixel (*left)[2], -// /*const*/ pixel *const top[2], int h, -// enum CdefEdgeFlags edges); +// void dav1d_cdef_paddingX_8bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, int h, +// enum CdefEdgeFlags edges); // n1 = s0/d0 // w1 = d0/q0 // n2 = s4/d2 // w2 = d2/q1 .macro padding_func w, stride, n1, w1, n2, w2, align -function cdef_padding\w\()_neon, export=1 +function cdef_padding\w\()_8bpc_neon, export=1 push {r4-r7,lr} ldrd r4, r5, [sp, #20] ldr r6, [sp, #28] + cmp r6, #0xf // fully edged + beq cdef_padding\w\()_edged_8bpc_neon vmov.i16 q3, #0x8000 tst r6, #4 // CDEF_HAVE_TOP bne 1f // !CDEF_HAVE_TOP sub r12, r0, #2*(2*\stride+2) @@ -173,14 +175,13 @@ vst1.16 {q2,q3}, [r12]! .endif b 3f 1: // CDEF_HAVE_TOP - ldr r7, [r4] - ldr lr, [r4, #4] + add r7, r4, r2 sub r0, r0, #2*(2*\stride) - pad_top_bottom r7, lr, \w, \stride, \n1, \w1, \n2, \w2, \align, 0 + pad_top_bottom r4, r7, \w, \stride, \n1, \w1, \n2, \w2, \align, 0 // Middle section 3: tst r6, #1 // CDEF_HAVE_LEFT beq 2f @@ -265,10 +266,69 @@ .endm padding_func 8, 16, d0, q0, d2, q1, 128 padding_func 4, 8, s0, d0, s4, d2, 64 +// void cdef_paddingX_edged_8bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, int h, +// enum CdefEdgeFlags edges); + +.macro padding_func_edged w, stride, reg, align +function cdef_padding\w\()_edged_8bpc_neon + sub r0, r0, #(2*\stride) + + ldrh r12, [r4, #-2] + vldr \reg, [r4] + add r7, r4, r2 + strh r12, [r0, #-2] + ldrh r12, [r4, #\w] + vstr \reg, [r0] + strh r12, [r0, #\w] + + ldrh r12, [r7, #-2] + vldr \reg, [r7] + strh r12, [r0, #\stride-2] + ldrh r12, [r7, #\w] + vstr \reg, [r0, #\stride] + strh r12, [r0, #\stride+\w] + add r0, r0, #2*\stride + +0: + ldrh r12, [r3], #2 + vldr \reg, [r1] + str r12, [r0, #-2] + ldrh r12, [r1, #\w] + add r1, r1, r2 + subs r5, r5, #1 + vstr \reg, [r0] + str r12, [r0, #\w] + add r0, r0, #\stride + bgt 0b + + ldrh r12, [r1, #-2] + vldr \reg, [r1] + add r7, r1, r2 + strh r12, [r0, #-2] + ldrh r12, [r1, #\w] + vstr \reg, [r0] + strh r12, [r0, #\w] + + ldrh r12, [r7, #-2] + vldr \reg, [r7] + strh r12, [r0, #\stride-2] + ldrh r12, [r7, #\w] + vstr \reg, [r0, #\stride] + strh r12, [r0, #\stride+\w] + + pop {r4-r7,pc} +endfunc +.endm + +padding_func_edged 8, 16, d0, 64 +padding_func_edged 4, 8, s0, 32 + .macro dir_table w, stride const directions\w .byte -1 * \stride + 1, -2 * \stride + 2 .byte 0 * \stride + 1, -1 * \stride + 2 .byte 0 * \stride + 1, 0 * \stride + 2 @@ -309,65 +369,72 @@ add r9, r9, #2*8 // += stride vld1.16 {\d12}, [r6] // p0 vld1.16 {\d22}, [r9] // p1 .endif .endm -.macro handle_pixel s1, s2, threshold, thresh_vec, shift, tap - cmp \threshold, #0 +.macro handle_pixel s1, s2, thresh_vec, shift, tap, min +.if \min vmin.u16 q2, q2, \s1 vmax.s16 q3, q3, \s1 vmin.u16 q2, q2, \s2 vmax.s16 q3, q3, \s2 - - beq 3f +.endif vabd.u16 q8, q0, \s1 // abs(diff) vabd.u16 q11, q0, \s2 // abs(diff) vshl.u16 q9, q8, \shift // abs(diff) >> shift vshl.u16 q12, q11, \shift // abs(diff) >> shift vqsub.u16 q9, \thresh_vec, q9 // clip = imax(0, threshold - (abs(diff) >> shift)) vqsub.u16 q12, \thresh_vec, q12// clip = imax(0, threshold - (abs(diff) >> shift)) vsub.i16 q10, \s1, q0 // diff = p0 - px - vsub.u16 q13, \s2, q0 // diff = p1 - px + vsub.i16 q13, \s2, q0 // diff = p1 - px vneg.s16 q8, q9 // -clip vneg.s16 q11, q12 // -clip vmin.s16 q10, q10, q9 // imin(diff, clip) vmin.s16 q13, q13, q12 // imin(diff, clip) vdup.16 q9, \tap // taps[k] vmax.s16 q10, q10, q8 // constrain() = imax(imin(diff, clip), -clip) vmax.s16 q13, q13, q11 // constrain() = imax(imin(diff, clip), -clip) vmla.i16 q1, q10, q9 // sum += taps[k] * constrain() vmla.i16 q1, q13, q9 // sum += taps[k] * constrain() -3: .endm -// void dav1d_cdef_filterX_neon(pixel *dst, ptrdiff_t dst_stride, -// const uint16_t *tmp, int pri_strength, -// int sec_strength, int dir, int damping, int h); -.macro filter w -function cdef_filter\w\()_neon, export=1 - push {r4-r9,lr} - vpush {q4-q7} - ldrd r4, r5, [sp, #92] - ldrd r6, r7, [sp, #100] +// void dav1d_cdef_filterX_8bpc_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint16_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h, size_t edges); +.macro filter_func w, pri, sec, min, suffix +function cdef_filter\w\suffix\()_neon + cmp r8, #0xf + beq cdef_filter\w\suffix\()_edged_neon +.if \pri movrel_local r8, pri_taps and r9, r3, #1 add r8, r8, r9, lsl #1 +.endif movrel_local r9, directions\w add r5, r9, r5, lsl #1 vmov.u16 d17, #15 vdup.16 d16, r6 // damping +.if \pri vdup.16 q5, r3 // threshold +.endif +.if \sec vdup.16 q7, r4 // threshold +.endif vmov.16 d8[0], r3 vmov.16 d8[1], r4 vclz.i16 d8, d8 // clz(threshold) vsub.i16 d8, d17, d8 // ulog2(threshold) vqsub.u16 d8, d16, d8 // shift = imax(0, damping - ulog2(threshold)) vneg.s16 d8, d8 // -shift +.if \sec vdup.16 q6, d8[1] +.endif +.if \pri vdup.16 q4, d8[0] +.endif 1: .if \w == 8 vld1.16 {q0}, [r2, :128] // px .else @@ -375,51 +442,68 @@ vld1.16 {d0}, [r2, :64] // px vld1.16 {d1}, [r12, :64] // px .endif vmov.u16 q1, #0 // sum +.if \min vmov.u16 q2, q0 // min vmov.u16 q3, q0 // max +.endif // Instead of loading sec_taps 2, 1 from memory, just set it // to 2 initially and decrease for the second round. + // This is also used as loop counter. mov lr, #2 // sec_taps[0] 2: +.if \pri ldrsb r9, [r5] // off1 load_px d28, d29, d30, d31, \w +.endif +.if \sec add r5, r5, #4 // +2*2 ldrsb r9, [r5] // off2 +.endif +.if \pri ldrb r12, [r8] // *pri_taps - handle_pixel q14, q15, r3, q5, q4, r12 + handle_pixel q14, q15, q5, q4, r12, \min +.endif +.if \sec load_px d28, d29, d30, d31, \w add r5, r5, #8 // +2*4 ldrsb r9, [r5] // off3 - handle_pixel q14, q15, r4, q7, q6, lr + handle_pixel q14, q15, q7, q6, lr, \min load_px d28, d29, d30, d31, \w - handle_pixel q14, q15, r4, q7, q6, lr + handle_pixel q14, q15, q7, q6, lr, \min - sub r5, r5, #11 // x8 -= 2*(2+4); x8 += 1; + sub r5, r5, #11 // r5 -= 2*(2+4); r5 += 1; +.else + add r5, r5, #1 // r5 += 1 +.endif subs lr, lr, #1 // sec_tap-- (value) +.if \pri add r8, r8, #1 // pri_taps++ (pointer) +.endif bne 2b vshr.s16 q14, q1, #15 // -(sum < 0) vadd.i16 q1, q1, q14 // sum - (sum < 0) vrshr.s16 q1, q1, #4 // (8 + sum - (sum < 0)) >> 4 vadd.i16 q0, q0, q1 // px + (8 + sum ...) >> 4 +.if \min vmin.s16 q0, q0, q3 vmax.s16 q0, q0, q2 // iclip(px + .., min, max) +.endif vmovn.u16 d0, q0 .if \w == 8 add r2, r2, #2*16 // tmp += tmp_stride subs r7, r7, #1 // h-- vst1.8 {d0}, [r0, :64], r1 @@ -428,34 +512,264 @@ add r2, r2, #2*16 // tmp += 2*tmp_stride subs r7, r7, #2 // h -= 2 vst1.32 {d0[1]}, [r0, :32], r1 .endif - // Reset pri_taps/sec_taps back to the original point + // Reset pri_taps and directions back to the original point sub r5, r5, #2 +.if \pri sub r8, r8, #2 +.endif bgt 1b vpop {q4-q7} pop {r4-r9,pc} endfunc .endm +.macro filter w +filter_func \w, pri=1, sec=0, min=0, suffix=_pri +filter_func \w, pri=0, sec=1, min=0, suffix=_sec +filter_func \w, pri=1, sec=1, min=1, suffix=_pri_sec + +function cdef_filter\w\()_8bpc_neon, export=1 + push {r4-r9,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #92] + ldrd r6, r7, [sp, #100] + ldr r8, [sp, #108] + cmp r3, #0 // pri_strength + bne 1f + b cdef_filter\w\()_sec_neon // only sec +1: + cmp r4, #0 // sec_strength + bne 1f + b cdef_filter\w\()_pri_neon // only pri +1: + b cdef_filter\w\()_pri_sec_neon // both pri and sec +endfunc +.endm + filter 8 filter 4 +.macro load_px_8 d11, d12, d21, d22, w +.if \w == 8 + add r6, r2, r9 // x + off + sub r9, r2, r9 // x - off + vld1.8 {\d11}, [r6] // p0 + add r6, r6, #16 // += stride + vld1.8 {\d21}, [r9] // p1 + add r9, r9, #16 // += stride + vld1.8 {\d12}, [r6] // p0 + vld1.8 {\d22}, [r9] // p1 +.else + add r6, r2, r9 // x + off + sub r9, r2, r9 // x - off + vld1.32 {\d11[0]}, [r6] // p0 + add r6, r6, #8 // += stride + vld1.32 {\d21[0]}, [r9] // p1 + add r9, r9, #8 // += stride + vld1.32 {\d11[1]}, [r6] // p0 + add r6, r6, #8 // += stride + vld1.32 {\d21[1]}, [r9] // p1 + add r9, r9, #8 // += stride + vld1.32 {\d12[0]}, [r6] // p0 + add r6, r6, #8 // += stride + vld1.32 {\d22[0]}, [r9] // p1 + add r9, r9, #8 // += stride + vld1.32 {\d12[1]}, [r6] // p0 + vld1.32 {\d22[1]}, [r9] // p1 +.endif +.endm +.macro handle_pixel_8 s1, s2, thresh_vec, shift, tap, min +.if \min + vmin.u8 q3, q3, \s1 + vmax.u8 q4, q4, \s1 + vmin.u8 q3, q3, \s2 + vmax.u8 q4, q4, \s2 +.endif + vabd.u8 q8, q0, \s1 // abs(diff) + vabd.u8 q11, q0, \s2 // abs(diff) + vshl.u8 q9, q8, \shift // abs(diff) >> shift + vshl.u8 q12, q11, \shift // abs(diff) >> shift + vqsub.u8 q9, \thresh_vec, q9 // clip = imax(0, threshold - (abs(diff) >> shift)) + vqsub.u8 q12, \thresh_vec, q12// clip = imax(0, threshold - (abs(diff) >> shift)) + vcgt.u8 q10, q0, \s1 // px > p0 + vcgt.u8 q13, q0, \s2 // px > p1 + vmin.u8 q9, q9, q8 // imin(abs(diff), clip) + vmin.u8 q12, q12, q11 // imin(abs(diff), clip) + vneg.s8 q8, q9 // -imin() + vneg.s8 q11, q12 // -imin() + vbsl q10, q8, q9 // constrain() = imax(imin(diff, clip), -clip) + vdup.8 d18, \tap // taps[k] + vbsl q13, q11, q12 // constrain() = imax(imin(diff, clip), -clip) + vmlal.s8 q1, d20, d18 // sum += taps[k] * constrain() + vmlal.s8 q1, d26, d18 // sum += taps[k] * constrain() + vmlal.s8 q2, d21, d18 // sum += taps[k] * constrain() + vmlal.s8 q2, d27, d18 // sum += taps[k] * constrain() +.endm + +// void cdef_filterX_edged_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint16_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h, size_t edges); +.macro filter_func_8 w, pri, sec, min, suffix +function cdef_filter\w\suffix\()_edged_neon +.if \pri + movrel_local r8, pri_taps + and r9, r3, #1 + add r8, r8, r9, lsl #1 +.endif + movrel_local r9, directions\w + add r5, r9, r5, lsl #1 + vmov.u8 d17, #7 + vdup.8 d16, r6 // damping + + vmov.8 d8[0], r3 + vmov.8 d8[1], r4 + vclz.i8 d8, d8 // clz(threshold) + vsub.i8 d8, d17, d8 // ulog2(threshold) + vqsub.u8 d8, d16, d8 // shift = imax(0, damping - ulog2(threshold)) + vneg.s8 d8, d8 // -shift +.if \sec + vdup.8 q6, d8[1] +.endif +.if \pri + vdup.8 q5, d8[0] +.endif + +1: +.if \w == 8 + add r12, r2, #16 + vld1.8 {d0}, [r2, :64] // px + vld1.8 {d1}, [r12, :64] // px +.else + add r12, r2, #8 + vld1.32 {d0[0]}, [r2, :32] // px + add r9, r2, #2*8 + vld1.32 {d0[1]}, [r12, :32] // px + add r12, r12, #2*8 + vld1.32 {d1[0]}, [r9, :32] // px + vld1.32 {d1[1]}, [r12, :32] // px +.endif + + vmov.u8 q1, #0 // sum + vmov.u8 q2, #0 // sum +.if \min + vmov.u16 q3, q0 // min + vmov.u16 q4, q0 // max +.endif + + // Instead of loading sec_taps 2, 1 from memory, just set it + // to 2 initially and decrease for the second round. + // This is also used as loop counter. + mov lr, #2 // sec_taps[0] + +2: +.if \pri + ldrsb r9, [r5] // off1 + + load_px_8 d28, d29, d30, d31, \w +.endif + +.if \sec + add r5, r5, #4 // +2*2 + ldrsb r9, [r5] // off2 +.endif + +.if \pri + ldrb r12, [r8] // *pri_taps + vdup.8 q7, r3 // threshold + + handle_pixel_8 q14, q15, q7, q5, r12, \min +.endif + +.if \sec + load_px_8 d28, d29, d30, d31, \w + + add r5, r5, #8 // +2*4 + ldrsb r9, [r5] // off3 + + vdup.8 q7, r4 // threshold + + handle_pixel_8 q14, q15, q7, q6, lr, \min + + load_px_8 d28, d29, d30, d31, \w + + handle_pixel_8 q14, q15, q7, q6, lr, \min + + sub r5, r5, #11 // r5 -= 2*(2+4); r5 += 1; +.else + add r5, r5, #1 // r5 += 1 +.endif + subs lr, lr, #1 // sec_tap-- (value) +.if \pri + add r8, r8, #1 // pri_taps++ (pointer) +.endif + bne 2b + + vshr.s16 q14, q1, #15 // -(sum < 0) + vshr.s16 q15, q2, #15 // -(sum < 0) + vadd.i16 q1, q1, q14 // sum - (sum < 0) + vadd.i16 q2, q2, q15 // sum - (sum < 0) + vrshr.s16 q1, q1, #4 // (8 + sum - (sum < 0)) >> 4 + vrshr.s16 q2, q2, #4 // (8 + sum - (sum < 0)) >> 4 + vaddw.u8 q1, q1, d0 // px + (8 + sum ...) >> 4 + vaddw.u8 q2, q2, d1 // px + (8 + sum ...) >> 4 + vqmovun.s16 d0, q1 + vqmovun.s16 d1, q2 +.if \min + vmin.u8 q0, q0, q4 + vmax.u8 q0, q0, q3 // iclip(px + .., min, max) +.endif +.if \w == 8 + vst1.8 {d0}, [r0, :64], r1 + add r2, r2, #2*16 // tmp += 2*tmp_stride + subs r7, r7, #2 // h -= 2 + vst1.8 {d1}, [r0, :64], r1 +.else + vst1.32 {d0[0]}, [r0, :32], r1 + add r2, r2, #4*8 // tmp += 4*tmp_stride + vst1.32 {d0[1]}, [r0, :32], r1 + subs r7, r7, #4 // h -= 4 + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d1[1]}, [r0, :32], r1 +.endif + + // Reset pri_taps and directions back to the original point + sub r5, r5, #2 +.if \pri + sub r8, r8, #2 +.endif + + bgt 1b + vpop {q4-q7} + pop {r4-r9,pc} +endfunc +.endm + +.macro filter_8 w +filter_func_8 \w, pri=1, sec=0, min=0, suffix=_pri +filter_func_8 \w, pri=0, sec=1, min=0, suffix=_sec +filter_func_8 \w, pri=1, sec=1, min=1, suffix=_pri_sec +.endm + +filter_8 8 +filter_8 4 + const div_table, align=4 .short 840, 420, 280, 210, 168, 140, 120, 105 endconst const alt_fact, align=4 .short 420, 210, 140, 105, 105, 105, 105, 105, 140, 210, 420, 0 endconst -// int dav1d_cdef_find_dir_neon(const pixel *img, const ptrdiff_t stride, -// unsigned *const var) -function cdef_find_dir_neon, export=1 +// int dav1d_cdef_find_dir_8bpc_neon(const pixel *img, const ptrdiff_t stride, +// unsigned *const var) +function cdef_find_dir_8bpc_neon, export=1 push {lr} vpush {q4-q7} sub sp, sp, #32 // cost mov r3, #8 vmov.u16 q1, #0 // q0-q1 sum_diag[0] diff --git a/third_party/dav1d/src/arm/32/ipred.S b/third_party/dav1d/src/arm/32/ipred.S --- a/third_party/dav1d/src/arm/32/ipred.S +++ b/third_party/dav1d/src/arm/32/ipred.S @@ -27,15 +27,15 @@ */ #include "src/arm/asm.S" #include "util.S" -// void ipred_dc_128_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_128_neon, export=1 +// void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_128_8bpc_neon, export=1 push {r4, lr} ldr r4, [sp, #8] clz r3, r3 adr r2, L(ipred_dc_128_tbl) sub r3, r3, #25 @@ -105,15 +105,15 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 64b pop {r4, pc} endfunc -// void ipred_v_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_v_neon, export=1 +// void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_8bpc_neon, export=1 push {r4, lr} ldr lr, [sp, #8] clz r3, r3 adr r4, L(ipred_v_tbl) sub r3, r3, #25 @@ -187,15 +187,15 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 64b pop {r4, pc} endfunc -// void ipred_h_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_h_neon, export=1 +// void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] clz r3, r3 adr r5, L(ipred_h_tbl) sub r3, r3, #25 @@ -295,15 +295,15 @@ vst1.8 {q3}, [r12, :128], r1 bgt 64b pop {r4-r5, pc} endfunc -// void ipred_dc_top_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_top_neon, export=1 +// void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] clz r3, r3 adr r5, L(ipred_dc_top_tbl) sub r3, r3, #25 @@ -416,15 +416,15 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 64b pop {r4-r5, pc} endfunc -// void ipred_dc_left_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_left_neon, export=1 +// void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] sub r2, r2, r4 clz r3, r3 clz lr, r4 @@ -554,15 +554,15 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 1b pop {r4-r5, pc} endfunc -// void ipred_dc_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_neon, export=1 +// void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_8bpc_neon, export=1 push {r4-r6, lr} ldr r4, [sp, #16] sub r2, r2, r4 add lr, r3, r4 // width + height clz r3, r3 @@ -763,14 +763,10 @@ vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w64): - vmov.8 q1, q0 - vmov.8 q2, q0 - vmov.8 q3, q0 -2: add r2, r2, #1 vld1.8 {d2, d3, d4, d5}, [r2]! vadd.s16 d0, d0, d30 vaddl.u8 q2, d4, d5 vaddl.u8 q1, d2, d3 diff --git a/third_party/dav1d/src/arm/32/loopfilter.S b/third_party/dav1d/src/arm/32/loopfilter.S --- a/third_party/dav1d/src/arm/32/loopfilter.S +++ b/third_party/dav1d/src/arm/32/loopfilter.S @@ -141,12 +141,12 @@ vmul.i16 q1, q1, q3 vmov.i8 d6, #4 vaddw.s8 q1, q1, d4 vmov.i8 d7, #3 vqmovn.s16 d2, q1 // f - vqadd.s8 d4, d6, d2 // imin(f + 4, 128) - vqadd.s8 d5, d7, d2 // imin(f + 3, 128) + vqadd.s8 d4, d6, d2 // imin(f + 4, 127) + vqadd.s8 d5, d7, d2 // imin(f + 3, 127) vshr.s8 d4, d4, #3 // f1 vshr.s8 d5, d5, #3 // f2 vmovl.u8 q1, d23 // p0 vmovl.u8 q3, d24 // q0 vaddw.s8 q1, q1, d5 @@ -732,17 +732,17 @@ vst1.32 {d25[1]}, [r0], r1 add r0, r0, #2 bx r12 endfunc -// void dav1d_lpf_v_sb_y_neon(pixel *dst, const ptrdiff_t stride, -// const uint32_t *const vmask, -// const uint8_t (*l)[4], ptrdiff_t b4_stride, -// const Av1FilterLUT *lut, const int w) +// void dav1d_lpf_v_sb_y_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint32_t *const vmask, +// const uint8_t (*l)[4], ptrdiff_t b4_stride, +// const Av1FilterLUT *lut, const int w) .macro lpf_func dir, type -function lpf_\dir\()_sb_\type\()_neon, export=1 +function lpf_\dir\()_sb_\type\()_8bpc_neon, export=1 push {r4-r11,lr} vpush {q4-q7} ldrd r4, r5, [sp, #100] ldrd r6, r7, [r2] // vmask[0], vmask[1] .ifc \type, y diff --git a/third_party/dav1d/src/arm/32/looprestoration.S b/third_party/dav1d/src/arm/32/looprestoration.S --- a/third_party/dav1d/src/arm/32/looprestoration.S +++ b/third_party/dav1d/src/arm/32/looprestoration.S @@ -26,15 +26,15 @@ */ #include "src/arm/asm.S" #include "util.S" -// void dav1d_wiener_filter_h_neon(int16_t *dst, const pixel (*left)[4], -// const pixel *src, ptrdiff_t stride, -// const int16_t fh[7], const intptr_t w, -// int h, enum LrEdgeFlags edges); -function wiener_filter_h_neon, export=1 +// void dav1d_wiener_filter_h_8bpc_neon(int16_t *dst, const pixel (*left)[4], +// const pixel *src, ptrdiff_t stride, +// const int16_t fh[7], const intptr_t w, +// int h, enum LrEdgeFlags edges); +function wiener_filter_h_8bpc_neon, export=1 push {r4-r11,lr} vpush {q4} ldrd r4, r5, [sp, #52] ldrd r6, r7, [sp, #60] mov r8, r5 @@ -365,15 +365,15 @@ pop {r4-r11,pc} .purgem filter_8 .purgem filter_4 endfunc -// void dav1d_wiener_filter_v_neon(pixel *dst, ptrdiff_t stride, -// const int16_t *mid, int w, int h, -// const int16_t fv[7], enum LrEdgeFlags edges, -// ptrdiff_t mid_stride); -function wiener_filter_v_neon, export=1 +// void dav1d_wiener_filter_v_8bpc_neon(pixel *dst, ptrdiff_t stride, +// const int16_t *mid, int w, int h, +// const int16_t fv[7], enum LrEdgeFlags edges, +// ptrdiff_t mid_stride); +function wiener_filter_v_8bpc_neon, export=1 push {r4-r7,lr} ldrd r4, r5, [sp, #20] ldrd r6, r7, [sp, #28] mov lr, r4 vmov.s16 q1, #0 @@ -546,13 +546,13 @@ 0: pop {r4-r7,pc} .purgem filter endfunc -// void dav1d_copy_narrow_neon(pixel *dst, ptrdiff_t stride, -// const pixel *src, int w, int h); -function copy_narrow_neon, export=1 +// void dav1d_copy_narrow_8bpc_neon(pixel *dst, ptrdiff_t stride, +// const pixel *src, int w, int h); +function copy_narrow_8bpc_neon, export=1 push {r4,lr} ldr r4, [sp, #8] adr r12, L(copy_narrow_tbl) ldr r3, [r12, r3, lsl #2] add r12, r12, r3 @@ -685,16 +685,16 @@ pop {r4,pc} endfunc #define SUM_STRIDE (384+16) -// void dav1d_sgr_box3_h_neon(int32_t *sumsq, int16_t *sum, -// const pixel (*left)[4], -// const pixel *src, const ptrdiff_t stride, -// const int w, const int h, -// const enum LrEdgeFlags edges); -function sgr_box3_h_neon, export=1 +// void dav1d_sgr_box3_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_h_8bpc_neon, export=1 push {r4-r11,lr} vpush {q4-q7} ldrd r4, r5, [sp, #100] ldrd r6, r7, [sp, #108] add r5, r5, #2 // w += 2 @@ -923,15 +923,15 @@ vmull.u8 q2, d1, d1 vmull.u8 q5, d8, d8 vmull.u8 q6, d9, d9 add3 4 + subs r5, r5, #4 vst1.16 {d6}, [r1, :64]! vst1.16 {d14}, [r11, :64]! vst1.32 {q12}, [r0, :128]! vst1.32 {q8}, [r10, :128]! - subs r5, r5, #4 ble 9f vext.8 q0, q0, q0, #4 vext.8 q1, q1, q2, #8 vext.8 q4, q4, q4, #4 vext.8 q5, q5, q6, #8 @@ -959,16 +959,16 @@ vpop {q4-q7} pop {r4-r11,pc} .purgem add3 endfunc -// void dav1d_sgr_box5_h_neon(int32_t *sumsq, int16_t *sum, -// const pixel (*left)[4], -// const pixel *src, const ptrdiff_t stride, -// const int w, const int h, -// const enum LrEdgeFlags edges); -function sgr_box5_h_neon, export=1 +// void dav1d_sgr_box5_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_h_8bpc_neon, export=1 push {r4-r11,lr} vpush {q4-q7} ldrd r4, r5, [sp, #100] ldrd r6, r7, [sp, #108] add r5, r5, #2 // w += 2 @@ -1036,11 +1036,11 @@ vext.8 q0, q1, q0, #13 vext.8 q4, q5, q4, #13 b 2f 0: // !LR_HAVE_LEFT, fill q1 with the leftmost byte - // and shift q0 to have 2x the first byte at the front. + // and shift q0 to have 3x the first byte at the front. vdup.8 q1, d0[0] vdup.8 q5, d8[0] // Move r3 back to account for the last 3 bytes we loaded before, // which we shifted out. sub r3, r3, #3 @@ -1213,15 +1213,15 @@ vmull.u8 q2, d1, d1 vmull.u8 q5, d8, d8 vmull.u8 q6, d9, d9 add5 4 + subs r5, r5, #4 vst1.16 {d6}, [r1, :64]! vst1.16 {d14}, [r11, :64]! vst1.32 {q12}, [r0, :128]! vst1.32 {q10}, [r10, :128]! - subs r5, r5, #4 ble 9f vext.8 q0, q0, q0, #4 vext.8 q1, q1, q2, #8 vext.8 q4, q4, q4, #4 vext.8 q5, q5, q6, #8 @@ -1659,15 +1659,15 @@ pop {r4-r5,pc} endfunc #define FILTER_OUT_STRIDE 384 -// void dav1d_sgr_finish_filter1_neon(coef *tmp, -// const pixel *src, const ptrdiff_t stride, -// const int32_t *a, const int16_t *b, -// const int w, const int h); -function sgr_finish_filter1_neon, export=1 +// void dav1d_sgr_finish_filter1_8bpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter1_8bpc_neon, export=1 push {r4-r11,lr} vpush {q4-q7} ldrd r4, r5, [sp, #100] ldr r6, [sp, #108] sub r7, r3, #(4*SUM_STRIDE) @@ -1763,15 +1763,15 @@ 0: vpop {q4-q7} pop {r4-r11,pc} endfunc -// void dav1d_sgr_finish_filter2_neon(coef *tmp, -// const pixel *src, const ptrdiff_t stride, -// const int32_t *a, const int16_t *b, -// const int w, const int h); -function sgr_finish_filter2_neon, export=1 +// void dav1d_sgr_finish_filter2_8bpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter2_8bpc_neon, export=1 push {r4-r11,lr} vpush {q4-q7} ldrd r4, r5, [sp, #100] ldr r6, [sp, #108] add r7, r3, #(4*(SUM_STRIDE)) @@ -1923,15 +1923,15 @@ 0: vpop {q4-q7} pop {r4-r11,pc} endfunc -// void dav1d_sgr_weighted1_neon(pixel *dst, const ptrdiff_t dst_stride, -// const pixel *src, const ptrdiff_t src_stride, -// const coef *t1, const int w, const int h, -// const int wt); -function sgr_weighted1_neon, export=1 +// void dav1d_sgr_weighted1_8bpc_neon(pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int w, const int h, +// const int wt); +function sgr_weighted1_8bpc_neon, export=1 push {r4-r9,lr} ldrd r4, r5, [sp, #28] ldrd r6, r7, [sp, #36] ldr r8, [sp, #44] vdup.16 d31, r7 @@ -2007,16 +2007,16 @@ bgt 2b 0: pop {r4-r9,pc} endfunc -// void dav1d_sgr_weighted2_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *src, const ptrdiff_t src_stride, -// const coef *t1, const coef *t2, -// const int w, const int h, -// const int16_t wt[2]); -function sgr_weighted2_neon, export=1 +// void dav1d_sgr_weighted2_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int16_t *t2, +// const int w, const int h, +// const int16_t wt[2]); +function sgr_weighted2_8bpc_neon, export=1 push {r4-r11,lr} ldrd r4, r5, [sp, #36] ldrd r6, r7, [sp, #44] ldr r8, [sp, #52] cmp r7, #2 diff --git a/third_party/dav1d/src/arm/32/mc.S b/third_party/dav1d/src/arm/32/mc.S --- a/third_party/dav1d/src/arm/32/mc.S +++ b/third_party/dav1d/src/arm/32/mc.S @@ -751,34 +751,32 @@ vmov.i8 d22, #64 vld1.32 {d4[]}, [r5, :32] add r12, r0, r1 lsl r1, r1, #1 vsub.i8 d5, d22, d4 - sub r1, r1, #3 + sub r1, r1, #2 4: vld1.u8 {d2}, [r2, :64]! vld1.32 {d0[]}, [r0, :32] vld1.32 {d0[1]}, [r12, :32] subs r4, r4, #2 vmull.u8 q3, d2, d4 vmlal.u8 q3, d0, d5 vrshrn.i16 d20, q3, #6 vst1.16 {d20[0]}, [r0, :16]! vst1.16 {d20[2]}, [r12, :16]! - vst1.8 {d20[2]}, [r0]! - vst1.8 {d20[6]}, [r12]! - add r0, r0, r1 - add r12, r12, r1 + vst1.8 {d20[2]}, [r0], r1 + vst1.8 {d20[6]}, [r12], r1 bgt 4b pop {r4-r5,pc} 80: vmov.i8 d16, #64 vld1.u8 {d2}, [r5, :64] add r12, r0, r1 lsl r1, r1, #1 vsub.i8 d17, d16, d2 - sub r1, r1, #6 + sub r1, r1, #4 8: vld1.u8 {d4, d5}, [r2, :128]! vld1.u8 {d0}, [r0, :64] vld1.u8 {d1}, [r12, :64] subs r4, r4, #2 @@ -788,23 +786,21 @@ vmlal.u8 q10, d1, d17 vrshrn.i16 d22, q3, #6 vrshrn.i16 d23, q10, #6 vst1.32 {d22[0]}, [r0, :32]! vst1.32 {d23[0]}, [r12, :32]! - vst1.16 {d22[2]}, [r0, :16]! - vst1.16 {d23[2]}, [r12, :16]! - add r0, r0, r1 - add r12, r12, r1 + vst1.16 {d22[2]}, [r0, :16], r1 + vst1.16 {d23[2]}, [r12, :16], r1 bgt 8b pop {r4-r5,pc} 160: vmov.i8 q12, #64 vld1.u8 {q14}, [r5, :128] add r12, r0, r1 lsl r1, r1, #1 vsub.i8 q11, q12, q14 - sub r1, r1, #12 + sub r1, r1, #8 16: vld1.u8 {q1, q2}, [r2, :128]! vld1.u8 {q0}, [r0, :128] subs r4, r4, #2 vld1.u8 {q13}, [r12, :128] @@ -820,24 +816,22 @@ vmlal.u8 q8, d27, d23 vrshrn.i16 d20, q3, #6 vrshrn.i16 d21, q8, #6 vst1.u8 {d18}, [r0, :64]! vst1.u8 {d20}, [r12, :64]! - vst1.32 {d19[0]}, [r0, :32]! - vst1.32 {d21[0]}, [r12, :32]! - add r0, r0, r1 - add r12, r12, r1 + vst1.32 {d19[0]}, [r0, :32], r1 + vst1.32 {d21[0]}, [r12, :32], r1 bgt 16b pop {r4-r5,pc} 320: vmov.i8 q10, #64 vld1.u8 {q2, q3}, [r5, :128] vsub.i8 q11, q10, q2 - vsub.i8 q12, q10, q3 + vsub.i8 d24, d20, d6 32: vld1.u8 {q8, q9}, [r2, :128]! - vld1.u8 {q0, q1}, [r0, :128] + vld1.u8 {d0, d1, d2}, [r0, :64] subs r4, r4, #1 vmull.u8 q15, d16, d4 vmlal.u8 q15, d0, d22 vmull.u8 q14, d17, d5 vmlal.u8 q14, d1, d23 @@ -3172,5 +3166,186 @@ endfunc .endm warp , 11 warp t, 7 + +// void dav1d_emu_edge_8bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_8bpc_neon, export=1 + push {r4-r11,lr} + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] + ldrd r8, r9, [sp, #52] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub r12, r3, #1 // ih - 1 + cmp r5, r3 + sub lr, r2, #1 // iw - 1 + it lt + movlt r12, r5 // min(y, ih - 1) + cmp r4, r2 + bic r12, r12, r12, asr #31 // max(min(y, ih - 1), 0) + it lt + movlt lr, r4 // min(x, iw - 1) + bic lr, lr, lr, asr #31 // max(min(x, iw - 1), 0) + mla r8, r12, r9, r8 // ref += iclip() * stride + add r8, r8, lr // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add r10, r5, r1 // y + bh + neg r5, r5 // -y + sub r10, r10, r3 // y + bh - ih + sub r12, r1, #1 // bh - 1 + cmp r10, r1 + bic r5, r5, r5, asr #31 // max(-y, 0) + it ge + movge r10, r12 // min(y + bh - ih, bh-1) + cmp r5, r1 + bic r10, r10, r10, asr #31 // max(min(y + bh - ih, bh-1), 0) + it ge + movge r5, r12 // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add r11, r4, r0 // x + bw + neg r4, r4 // -x + sub r11, r11, r2 // x + bw - iw + sub lr, r0, #1 // bw - 1 + cmp r11, r0 + bic r4, r4, r4, asr #31 // max(-x, 0) + it ge + movge r11, lr // min(x + bw - iw, bw-1) + cmp r4, r0 + bic r11, r11, r11, asr #31 // max(min(x + bw - iw, bw-1), 0) + it ge + movge r4, lr // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub r1, r1, r5 // bh - top_ext + mla r6, r5, r7, r6 + sub r2, r0, r4 // bw - left_ext + sub r1, r1, r10 // center_h = bh - top_ext - bottom_ext + sub r2, r2, r11 // center_w = bw - left_ext - right_ext + + mov r0, r6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + vld1.8 {d0[]}, [r8] + mov r12, r6 // out = dst + mov r3, r4 + vmov d1, d0 +1: + subs r3, r3, #16 + vst1.8 {q0}, [r12, :128]! + bgt 1b +.endif + mov lr, r8 + add r12, r6, r4 // out = dst + left_ext + mov r3, r2 +1: + vld1.8 {q0, q1}, [lr]! + subs r3, r3, #32 +.if \need_left + vst1.8 {q0, q1}, [r12]! +.else + vst1.8 {q0, q1}, [r12, :128]! +.endif + bgt 1b +.if \need_right + add r3, r8, r2 // in + center_w + sub r3, r3, #1 // in + center_w - 1 + add r12, r6, r4 // dst + left_ext + vld1.8 {d0[]}, [r3] + add r12, r12, r2 // out = dst + left_ext + center_w + mov r3, r11 + vmov d1, d0 +1: + subs r3, r3, #16 + vst1.8 {q0}, [r12]! + bgt 1b +.endif + + subs r1, r1, #1 // center_h-- + add r6, r6, r7 + add r8, r8, r9 + bgt 0b +.endm + + cmp r4, #0 + beq 2f + // need_left + cmp r11, #0 + beq 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cmp r11, #0 + beq 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + cmp r10, #0 + // Storing the original dst in r0 overwrote bw, recalculate it here + add r2, r2, r4 // center_w + left_ext + add r2, r2, r11 // bw = center_w + left_ext + right_ext + + beq 3f + // need_bottom + sub r8, r6, r7 // ref = dst - stride + mov r4, r2 +1: + vld1.8 {q0, q1}, [r8, :128]! + mov r3, r10 +2: + subs r3, r3, #1 + vst1.8 {q0, q1}, [r6, :128], r7 + bgt 2b + mls r6, r7, r10, r6 // dst -= bottom_ext * stride + subs r4, r4, #32 // bw -= 32 + add r6, r6, #32 // dst += 32 + bgt 1b + +3: + cmp r5, #0 + beq 3f + // need_top + mls r6, r7, r5, r0 // dst = stored_dst - top_ext * stride +1: + vld1.8 {q0, q1}, [r0, :128]! + mov r3, r5 +2: + subs r3, r3, #1 + vst1.8 {q0, q1}, [r6, :128], r7 + bgt 2b + mls r6, r7, r5, r6 // dst -= top_ext * stride + subs r2, r2, #32 // bw -= 32 + add r6, r6, #32 // dst += 32 + bgt 1b + +3: + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/64/cdef.S b/third_party/dav1d/src/arm/64/cdef.S --- a/third_party/dav1d/src/arm/64/cdef.S +++ b/third_party/dav1d/src/arm/64/cdef.S @@ -25,10 +25,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/arm/asm.S" #include "util.S" +#include "cdef_tmpl.S" .macro pad_top_bottom s1, s2, w, stride, rn, rw, ret tst w6, #1 // CDEF_HAVE_LEFT b.eq 2f // CDEF_HAVE_LEFT @@ -135,17 +136,19 @@ .else ld1 {\dst\().8b}, [\src], \incr .endif .endm -// void dav1d_cdef_paddingX_neon(uint16_t *tmp, const pixel *src, -// ptrdiff_t src_stride, const pixel (*left)[2], -// /*const*/ pixel *const top[2], int h, -// enum CdefEdgeFlags edges); +// void dav1d_cdef_paddingX_8bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, int h, +// enum CdefEdgeFlags edges); .macro padding_func w, stride, rn, rw -function cdef_padding\w\()_neon, export=1 +function cdef_padding\w\()_8bpc_neon, export=1 + cmp w6, #0xf // fully edged + b.eq cdef_padding\w\()_edged_8bpc_neon movi v30.8h, #0x80, lsl #8 mov v31.16b, v30.16b sub x0, x0, #2*(2*\stride+2) tst w6, #4 // CDEF_HAVE_TOP b.ne 1f @@ -155,13 +158,12 @@ st1 {v30.8h, v31.8h}, [x0], #32 .endif b 3f 1: // CDEF_HAVE_TOP - ldr x8, [x4] - ldr x9, [x4, #8] - pad_top_bottom x8, x9, \w, \stride, \rn, \rw, 0 + add x9, x4, x2 + pad_top_bottom x4, x9, \w, \stride, \rn, \rw, 0 // Middle section 3: tst w6, #1 // CDEF_HAVE_LEFT b.eq 2f @@ -240,360 +242,276 @@ .endm padding_func 8, 16, d, q padding_func 4, 8, s, d -.macro dir_table w, stride -const directions\w - .byte -1 * \stride + 1, -2 * \stride + 2 - .byte 0 * \stride + 1, -1 * \stride + 2 - .byte 0 * \stride + 1, 0 * \stride + 2 - .byte 0 * \stride + 1, 1 * \stride + 2 - .byte 1 * \stride + 1, 2 * \stride + 2 - .byte 1 * \stride + 0, 2 * \stride + 1 - .byte 1 * \stride + 0, 2 * \stride + 0 - .byte 1 * \stride + 0, 2 * \stride - 1 -// Repeated, to avoid & 7 - .byte -1 * \stride + 1, -2 * \stride + 2 - .byte 0 * \stride + 1, -1 * \stride + 2 - .byte 0 * \stride + 1, 0 * \stride + 2 - .byte 0 * \stride + 1, 1 * \stride + 2 - .byte 1 * \stride + 1, 2 * \stride + 2 - .byte 1 * \stride + 0, 2 * \stride + 1 -endconst +// void cdef_paddingX_edged_8bpc_neon(uint8_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, int h, +// enum CdefEdgeFlags edges); + +.macro padding_func_edged w, stride, reg +function cdef_padding\w\()_edged_8bpc_neon, export=1 + sub x4, x4, #2 + sub x0, x0, #(2*\stride+2) + +.if \w == 4 + ldr d0, [x4] + ldr d1, [x4, x2] + st1 {v0.8b, v1.8b}, [x0], #16 +.else + add x9, x4, x2 + ldr d0, [x4] + ldr s1, [x4, #8] + ldr d2, [x9] + ldr s3, [x9, #8] + str d0, [x0] + str s1, [x0, #8] + str d2, [x0, #\stride] + str s3, [x0, #\stride+8] + add x0, x0, #2*\stride +.endif + +0: + ld1 {v0.h}[0], [x3], #2 + ldr h2, [x1, #\w] + load_n_incr v1, x1, x2, \w + subs w5, w5, #1 + str h0, [x0] + stur \reg\()1, [x0, #2] + str h2, [x0, #2+\w] + add x0, x0, #\stride + b.gt 0b + + sub x1, x1, #2 +.if \w == 4 + ldr d0, [x1] + ldr d1, [x1, x2] + st1 {v0.8b, v1.8b}, [x0], #16 +.else + add x9, x1, x2 + ldr d0, [x1] + ldr s1, [x1, #8] + ldr d2, [x9] + ldr s3, [x9, #8] + str d0, [x0] + str s1, [x0, #8] + str d2, [x0, #\stride] + str s3, [x0, #\stride+8] +.endif + ret +endfunc .endm -dir_table 8, 16 -dir_table 4, 8 +padding_func_edged 8, 16, d +padding_func_edged 4, 8, s + +tables -const pri_taps - .byte 4, 2, 3, 3 -endconst +filter 8, 8 +filter 4, 8 -.macro load_px d1, d2, w +find_dir 8 + +.macro load_px_8 d1, d2, w .if \w == 8 - add x6, x2, w9, sxtb #1 // x + off - sub x9, x2, w9, sxtb #1 // x - off - ld1 {\d1\().8h}, [x6] // p0 - ld1 {\d2\().8h}, [x9] // p1 + add x6, x2, w9, sxtb // x + off + sub x9, x2, w9, sxtb // x - off + ld1 {\d1\().d}[0], [x6] // p0 + add x6, x6, #16 // += stride + ld1 {\d2\().d}[0], [x9] // p1 + add x9, x9, #16 // += stride + ld1 {\d1\().d}[1], [x6] // p0 + ld1 {\d2\().d}[1], [x9] // p0 .else - add x6, x2, w9, sxtb #1 // x + off - sub x9, x2, w9, sxtb #1 // x - off - ld1 {\d1\().4h}, [x6] // p0 - add x6, x6, #2*8 // += stride - ld1 {\d2\().4h}, [x9] // p1 - add x9, x9, #2*8 // += stride - ld1 {\d1\().d}[1], [x6] // p0 - ld1 {\d2\().d}[1], [x9] // p1 + add x6, x2, w9, sxtb // x + off + sub x9, x2, w9, sxtb // x - off + ld1 {\d1\().s}[0], [x6] // p0 + add x6, x6, #8 // += stride + ld1 {\d2\().s}[0], [x9] // p1 + add x9, x9, #8 // += stride + ld1 {\d1\().s}[1], [x6] // p0 + add x6, x6, #8 // += stride + ld1 {\d2\().s}[1], [x9] // p1 + add x9, x9, #8 // += stride + ld1 {\d1\().s}[2], [x6] // p0 + add x6, x6, #8 // += stride + ld1 {\d2\().s}[2], [x9] // p1 + add x9, x9, #8 // += stride + ld1 {\d1\().s}[3], [x6] // p0 + ld1 {\d2\().s}[3], [x9] // p1 .endif .endm -.macro handle_pixel s1, s2, threshold, thresh_vec, shift, tap - umin v2.8h, v2.8h, \s1\().8h - smax v3.8h, v3.8h, \s1\().8h - umin v2.8h, v2.8h, \s2\().8h - smax v3.8h, v3.8h, \s2\().8h - - cbz \threshold, 3f - uabd v16.8h, v0.8h, \s1\().8h // abs(diff) - uabd v20.8h, v0.8h, \s2\().8h // abs(diff) - ushl v17.8h, v16.8h, \shift // abs(diff) >> shift - ushl v21.8h, v20.8h, \shift // abs(diff) >> shift - uqsub v17.8h, \thresh_vec, v17.8h // clip = imax(0, threshold - (abs(diff) >> shift)) - uqsub v21.8h, \thresh_vec, v21.8h // clip = imax(0, threshold - (abs(diff) >> shift)) - sub v18.8h, \s1\().8h, v0.8h // diff = p0 - px - sub v22.8h, \s2\().8h, v0.8h // diff = p1 - px - neg v16.8h, v17.8h // -clip - neg v20.8h, v21.8h // -clip - smin v18.8h, v18.8h, v17.8h // imin(diff, clip) - smin v22.8h, v22.8h, v21.8h // imin(diff, clip) - dup v19.8h, \tap // taps[k] - smax v18.8h, v18.8h, v16.8h // constrain() = imax(imin(diff, clip), -clip) - smax v22.8h, v22.8h, v20.8h // constrain() = imax(imin(diff, clip), -clip) - mla v1.8h, v18.8h, v19.8h // sum += taps[k] * constrain() - mla v1.8h, v22.8h, v19.8h // sum += taps[k] * constrain() -3: +.macro handle_pixel_8 s1, s2, thresh_vec, shift, tap, min +.if \min + umin v3.16b, v3.16b, \s1\().16b + umax v4.16b, v4.16b, \s1\().16b + umin v3.16b, v3.16b, \s2\().16b + umax v4.16b, v4.16b, \s2\().16b +.endif + uabd v16.16b, v0.16b, \s1\().16b // abs(diff) + uabd v20.16b, v0.16b, \s2\().16b // abs(diff) + ushl v17.16b, v16.16b, \shift // abs(diff) >> shift + ushl v21.16b, v20.16b, \shift // abs(diff) >> shift + uqsub v17.16b, \thresh_vec, v17.16b // clip = imax(0, threshold - (abs(diff) >> shift)) + uqsub v21.16b, \thresh_vec, v21.16b // clip = imax(0, threshold - (abs(diff) >> shift)) + cmhi v18.16b, v0.16b, \s1\().16b // px > p0 + cmhi v22.16b, v0.16b, \s2\().16b // px > p1 + umin v17.16b, v17.16b, v16.16b // imin(abs(diff), clip) + umin v21.16b, v21.16b, v20.16b // imin(abs(diff), clip) + dup v19.16b, \tap // taps[k] + neg v16.16b, v17.16b // -imin() + neg v20.16b, v21.16b // -imin() + bsl v18.16b, v16.16b, v17.16b // constrain() = apply_sign() + bsl v22.16b, v20.16b, v21.16b // constrain() = apply_sign() + smlal v1.8h, v18.8b, v19.8b // sum += taps[k] * constrain() + smlal v1.8h, v22.8b, v19.8b // sum += taps[k] * constrain() + smlal2 v2.8h, v18.16b, v19.16b // sum += taps[k] * constrain() + smlal2 v2.8h, v22.16b, v19.16b // sum += taps[k] * constrain() .endm -// void dav1d_cdef_filterX_neon(pixel *dst, ptrdiff_t dst_stride, -// const uint16_t *tmp, int pri_strength, -// int sec_strength, int dir, int damping, int h); -.macro filter w -function cdef_filter\w\()_neon, export=1 +// void cdef_filterX_edged_8bpc_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint8_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h); +.macro filter_func_8 w, pri, sec, min, suffix +function cdef_filter\w\suffix\()_edged_8bpc_neon +.if \pri movrel x8, pri_taps and w9, w3, #1 add x8, x8, w9, uxtw #1 +.endif movrel x9, directions\w add x5, x9, w5, uxtw #1 - movi v30.4h, #15 - dup v28.4h, w6 // damping + movi v30.8b, #7 + dup v28.8b, w6 // damping - dup v25.8h, w3 // threshold - dup v27.8h, w4 // threshold - trn1 v24.4h, v25.4h, v27.4h - clz v24.4h, v24.4h // clz(threshold) - sub v24.4h, v30.4h, v24.4h // ulog2(threshold) - uqsub v24.4h, v28.4h, v24.4h // shift = imax(0, damping - ulog2(threshold)) - neg v24.4h, v24.4h // -shift - dup v26.8h, v24.h[1] - dup v24.8h, v24.h[0] +.if \pri + dup v25.16b, w3 // threshold +.endif +.if \sec + dup v27.16b, w4 // threshold +.endif + trn1 v24.8b, v25.8b, v27.8b + clz v24.8b, v24.8b // clz(threshold) + sub v24.8b, v30.8b, v24.8b // ulog2(threshold) + uqsub v24.8b, v28.8b, v24.8b // shift = imax(0, damping - ulog2(threshold)) + neg v24.8b, v24.8b // -shift +.if \sec + dup v26.16b, v24.b[1] +.endif +.if \pri + dup v24.16b, v24.b[0] +.endif 1: .if \w == 8 - ld1 {v0.8h}, [x2] // px + add x12, x2, #16 + ld1 {v0.d}[0], [x2] // px + ld1 {v0.d}[1], [x12] // px .else - add x12, x2, #2*8 - ld1 {v0.4h}, [x2] // px - ld1 {v0.d}[1], [x12] // px + add x12, x2, #1*8 + add x13, x2, #2*8 + add x14, x2, #3*8 + ld1 {v0.s}[0], [x2] // px + ld1 {v0.s}[1], [x12] // px + ld1 {v0.s}[2], [x13] // px + ld1 {v0.s}[3], [x14] // px .endif movi v1.8h, #0 // sum - mov v2.16b, v0.16b // min - mov v3.16b, v0.16b // max + movi v2.8h, #0 // sum +.if \min + mov v3.16b, v0.16b // min + mov v4.16b, v0.16b // max +.endif // Instead of loading sec_taps 2, 1 from memory, just set it // to 2 initially and decrease for the second round. + // This is also used as loop counter. mov w11, #2 // sec_taps[0] 2: +.if \pri ldrb w9, [x5] // off1 - load_px v4, v5, \w + load_px_8 v5, v6, \w +.endif +.if \sec add x5, x5, #4 // +2*2 ldrb w9, [x5] // off2 - load_px v6, v7, \w + load_px_8 v28, v29, \w +.endif +.if \pri ldrb w10, [x8] // *pri_taps - handle_pixel v4, v5, w3, v25.8h, v24.8h, w10 + handle_pixel_8 v5, v6, v25.16b, v24.16b, w10, \min +.endif +.if \sec add x5, x5, #8 // +2*4 ldrb w9, [x5] // off3 - load_px v4, v5, \w + load_px_8 v5, v6, \w - handle_pixel v6, v7, w4, v27.8h, v26.8h, w11 + handle_pixel_8 v28, v29, v27.16b, v26.16b, w11, \min + + handle_pixel_8 v5, v6, v27.16b, v26.16b, w11, \min - handle_pixel v4, v5, w4, v27.8h, v26.8h, w11 - - sub x5, x5, #11 // x8 -= 2*(2+4); x8 += 1; + sub x5, x5, #11 // x5 -= 2*(2+4); x5 += 1; +.else + add x5, x5, #1 // x5 += 1 +.endif subs w11, w11, #1 // sec_tap-- (value) +.if \pri add x8, x8, #1 // pri_taps++ (pointer) +.endif b.ne 2b - sshr v4.8h, v1.8h, #15 // -(sum < 0) - add v1.8h, v1.8h, v4.8h // sum - (sum < 0) - srshr v1.8h, v1.8h, #4 // (8 + sum - (sum < 0)) >> 4 - add v0.8h, v0.8h, v1.8h // px + (8 + sum ...) >> 4 - smin v0.8h, v0.8h, v3.8h - smax v0.8h, v0.8h, v2.8h // iclip(px + .., min, max) - xtn v0.8b, v0.8h + sshr v5.8h, v1.8h, #15 // -(sum < 0) + sshr v6.8h, v2.8h, #15 // -(sum < 0) + add v1.8h, v1.8h, v5.8h // sum - (sum < 0) + add v2.8h, v2.8h, v6.8h // sum - (sum < 0) + srshr v1.8h, v1.8h, #4 // (8 + sum - (sum < 0)) >> 4 + srshr v2.8h, v2.8h, #4 // (8 + sum - (sum < 0)) >> 4 + uaddw v1.8h, v1.8h, v0.8b // px + (8 + sum ...) >> 4 + uaddw2 v2.8h, v2.8h, v0.16b // px + (8 + sum ...) >> 4 + sqxtun v0.8b, v1.8h + sqxtun2 v0.16b, v2.8h +.if \min + umin v0.16b, v0.16b, v4.16b + umax v0.16b, v0.16b, v3.16b // iclip(px + .., min, max) +.endif .if \w == 8 - add x2, x2, #2*16 // tmp += tmp_stride - subs w7, w7, #1 // h-- - st1 {v0.8b}, [x0], x1 + st1 {v0.d}[0], [x0], x1 + add x2, x2, #2*16 // tmp += 2*tmp_stride + subs w7, w7, #2 // h -= 2 + st1 {v0.d}[1], [x0], x1 .else st1 {v0.s}[0], [x0], x1 - add x2, x2, #2*16 // tmp += 2*tmp_stride - subs w7, w7, #2 // h -= 2 + add x2, x2, #4*8 // tmp += 4*tmp_stride st1 {v0.s}[1], [x0], x1 + subs w7, w7, #4 // h -= 4 + st1 {v0.s}[2], [x0], x1 + st1 {v0.s}[3], [x0], x1 .endif - // Reset pri_taps/sec_taps back to the original point + // Reset pri_taps and directions back to the original point sub x5, x5, #2 +.if \pri sub x8, x8, #2 +.endif b.gt 1b ret endfunc .endm -filter 8 -filter 4 - -const div_table - .short 840, 420, 280, 210, 168, 140, 120, 105 -endconst - -const alt_fact - .short 420, 210, 140, 105, 105, 105, 105, 105, 140, 210, 420, 0 -endconst - -// int dav1d_cdef_find_dir_neon(const pixel *img, const ptrdiff_t stride, -// unsigned *const var) -function cdef_find_dir_neon, export=1 - sub sp, sp, #32 // cost - mov w3, #8 - movi v31.16b, #128 - movi v30.16b, #0 - movi v1.8h, #0 // v0-v1 sum_diag[0] - movi v3.8h, #0 // v2-v3 sum_diag[1] - movi v5.8h, #0 // v4-v5 sum_hv[0-1] - movi v7.8h, #0 // v6-v7 sum_alt[0] - movi v17.8h, #0 // v16-v17 sum_alt[1] - movi v18.8h, #0 // v18-v19 sum_alt[2] - movi v19.8h, #0 - movi v21.8h, #0 // v20-v21 sum_alt[3] - -.irpc i, 01234567 - ld1 {v26.8b}, [x0], x1 - usubl v26.8h, v26.8b, v31.8b - - addv h25, v26.8h // [y] - rev64 v27.8h, v26.8h - addp v28.8h, v26.8h, v30.8h // [(x >> 1)] - add v5.8h, v5.8h, v26.8h // sum_hv[1] - ext v27.16b, v27.16b, v27.16b, #8 // [-x] - rev64 v29.4h, v28.4h // [-(x >> 1)] - ins v4.h[\i], v25.h[0] // sum_hv[0] - -.if \i == 0 - mov v0.16b, v26.16b // sum_diag[0] - mov v2.16b, v27.16b // sum_diag[1] - mov v6.16b, v28.16b // sum_alt[0] - mov v16.16b, v29.16b // sum_alt[1] -.else - ext v22.16b, v30.16b, v26.16b, #(16-2*\i) - ext v23.16b, v26.16b, v30.16b, #(16-2*\i) - ext v24.16b, v30.16b, v27.16b, #(16-2*\i) - ext v25.16b, v27.16b, v30.16b, #(16-2*\i) - add v0.8h, v0.8h, v22.8h // sum_diag[0] - add v1.8h, v1.8h, v23.8h // sum_diag[0] - add v2.8h, v2.8h, v24.8h // sum_diag[1] - add v3.8h, v3.8h, v25.8h // sum_diag[1] - ext v22.16b, v30.16b, v28.16b, #(16-2*\i) - ext v23.16b, v28.16b, v30.16b, #(16-2*\i) - ext v24.16b, v30.16b, v29.16b, #(16-2*\i) - ext v25.16b, v29.16b, v30.16b, #(16-2*\i) - add v6.8h, v6.8h, v22.8h // sum_alt[0] - add v7.4h, v7.4h, v23.4h // sum_alt[0] - add v16.8h, v16.8h, v24.8h // sum_alt[1] - add v17.4h, v17.4h, v25.4h // sum_alt[1] -.endif -.if \i < 6 - ext v22.16b, v30.16b, v26.16b, #(16-2*(3-(\i/2))) - ext v23.16b, v26.16b, v30.16b, #(16-2*(3-(\i/2))) - add v18.8h, v18.8h, v22.8h // sum_alt[2] - add v19.4h, v19.4h, v23.4h // sum_alt[2] -.else - add v18.8h, v18.8h, v26.8h // sum_alt[2] -.endif -.if \i == 0 - mov v20.16b, v26.16b // sum_alt[3] -.elseif \i == 1 - add v20.8h, v20.8h, v26.8h // sum_alt[3] -.else - ext v24.16b, v30.16b, v26.16b, #(16-2*(\i/2)) - ext v25.16b, v26.16b, v30.16b, #(16-2*(\i/2)) - add v20.8h, v20.8h, v24.8h // sum_alt[3] - add v21.4h, v21.4h, v25.4h // sum_alt[3] -.endif -.endr - - movi v31.4s, #105 - - smull v26.4s, v4.4h, v4.4h // sum_hv[0]*sum_hv[0] - smlal2 v26.4s, v4.8h, v4.8h - smull v27.4s, v5.4h, v5.4h // sum_hv[1]*sum_hv[1] - smlal2 v27.4s, v5.8h, v5.8h - mul v26.4s, v26.4s, v31.4s // cost[2] *= 105 - mul v27.4s, v27.4s, v31.4s // cost[6] *= 105 - addv s4, v26.4s // cost[2] - addv s5, v27.4s // cost[6] +.macro filter_8 w +filter_func_8 \w, pri=1, sec=0, min=0, suffix=_pri +filter_func_8 \w, pri=0, sec=1, min=0, suffix=_sec +filter_func_8 \w, pri=1, sec=1, min=1, suffix=_pri_sec +.endm - rev64 v1.8h, v1.8h - rev64 v3.8h, v3.8h - ext v1.16b, v1.16b, v1.16b, #10 // sum_diag[0][14-n] - ext v3.16b, v3.16b, v3.16b, #10 // sum_diag[1][14-n] - - str s4, [sp, #2*4] // cost[2] - str s5, [sp, #6*4] // cost[6] - - movrel x4, div_table - ld1 {v31.8h}, [x4] - - smull v22.4s, v0.4h, v0.4h // sum_diag[0]*sum_diag[0] - smull2 v23.4s, v0.8h, v0.8h - smlal v22.4s, v1.4h, v1.4h - smlal2 v23.4s, v1.8h, v1.8h - smull v24.4s, v2.4h, v2.4h // sum_diag[1]*sum_diag[1] - smull2 v25.4s, v2.8h, v2.8h - smlal v24.4s, v3.4h, v3.4h - smlal2 v25.4s, v3.8h, v3.8h - uxtl v30.4s, v31.4h // div_table - uxtl2 v31.4s, v31.8h - mul v22.4s, v22.4s, v30.4s // cost[0] - mla v22.4s, v23.4s, v31.4s // cost[0] - mul v24.4s, v24.4s, v30.4s // cost[4] - mla v24.4s, v25.4s, v31.4s // cost[4] - addv s0, v22.4s // cost[0] - addv s2, v24.4s // cost[4] - - movrel x5, alt_fact - ld1 {v29.4h, v30.4h, v31.4h}, [x5]// div_table[2*m+1] + 105 - - str s0, [sp, #0*4] // cost[0] - str s2, [sp, #4*4] // cost[4] - - uxtl v29.4s, v29.4h // div_table[2*m+1] + 105 - uxtl v30.4s, v30.4h - uxtl v31.4s, v31.4h - -.macro cost_alt d1, d2, s1, s2, s3, s4 - smull v22.4s, \s1\().4h, \s1\().4h // sum_alt[n]*sum_alt[n] - smull2 v23.4s, \s1\().8h, \s1\().8h - smull v24.4s, \s2\().4h, \s2\().4h - smull v25.4s, \s3\().4h, \s3\().4h // sum_alt[n]*sum_alt[n] - smull2 v26.4s, \s3\().8h, \s3\().8h - smull v27.4s, \s4\().4h, \s4\().4h - mul v22.4s, v22.4s, v29.4s // sum_alt[n]^2*fact - mla v22.4s, v23.4s, v30.4s - mla v22.4s, v24.4s, v31.4s - mul v25.4s, v25.4s, v29.4s // sum_alt[n]^2*fact - mla v25.4s, v26.4s, v30.4s - mla v25.4s, v27.4s, v31.4s - addv \d1, v22.4s // *cost_ptr - addv \d2, v25.4s // *cost_ptr -.endm - cost_alt s6, s16, v6, v7, v16, v17 // cost[1], cost[3] - cost_alt s18, s20, v18, v19, v20, v21 // cost[5], cost[7] - str s6, [sp, #1*4] // cost[1] - str s16, [sp, #3*4] // cost[3] - - mov w0, #0 // best_dir - mov w1, v0.s[0] // best_cost - mov w3, #1 // n - - str s18, [sp, #5*4] // cost[5] - str s20, [sp, #7*4] // cost[7] - - mov w4, v6.s[0] - -.macro find_best s1, s2, s3 -.ifnb \s2 - mov w5, \s2\().s[0] -.endif - cmp w4, w1 // cost[n] > best_cost - csel w0, w3, w0, gt // best_dir = n - csel w1, w4, w1, gt // best_cost = cost[n] -.ifnb \s2 - add w3, w3, #1 // n++ - cmp w5, w1 // cost[n] > best_cost - mov w4, \s3\().s[0] - csel w0, w3, w0, gt // best_dir = n - csel w1, w5, w1, gt // best_cost = cost[n] - add w3, w3, #1 // n++ -.endif -.endm - find_best v6, v4, v16 - find_best v16, v2, v18 - find_best v18, v5, v20 - find_best v20 - - eor w3, w0, #4 // best_dir ^4 - ldr w4, [sp, w3, uxtw #2] - sub w1, w1, w4 // best_cost - cost[best_dir ^ 4] - lsr w1, w1, #10 - str w1, [x2] // *var - - add sp, sp, #32 - ret -endfunc +filter_8 8 +filter_8 4 diff --git a/third_party/dav1d/src/arm/64/cdef16.S b/third_party/dav1d/src/arm/64/cdef16.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/cdef16.S @@ -0,0 +1,228 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "cdef_tmpl.S" + +.macro pad_top_bot_16 s1, s2, w, stride, reg, ret + tst w6, #1 // CDEF_HAVE_LEFT + b.eq 2f + // CDEF_HAVE_LEFT + sub \s1, \s1, #4 + sub \s2, \s2, #4 + tst w6, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr d1, [\s1, #2*\w] + ldr \reg\()2, [\s2] + ldr d3, [\s2, #2*\w] + str \reg\()0, [x0] + str d1, [x0, #2*\w] + add x0, x0, #2*\stride + str \reg\()2, [x0] + str d3, [x0, #2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr s1, [\s1, #2*\w] + ldr \reg\()2, [\s2] + ldr s3, [\s2, #2*\w] + str \reg\()0, [x0] + str s1, [x0, #2*\w] + str s31, [x0, #2*\w+4] + add x0, x0, #2*\stride + str \reg\()2, [x0] + str s3, [x0, #2*\w] + str s31, [x0, #2*\w+4] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +2: + // !CDEF_HAVE_LEFT + tst w6, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr s1, [\s1, #2*\w] + ldr \reg\()2, [\s2] + ldr s3, [\s2, #2*\w] + str s31, [x0] + stur \reg\()0, [x0, #4] + str s1, [x0, #4+2*\w] + add x0, x0, #2*\stride + str s31, [x0] + stur \reg\()2, [x0, #4] + str s3, [x0, #4+2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr \reg\()1, [\s2] + str s31, [x0] + stur \reg\()0, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + str s31, [x0] + stur \reg\()1, [x0, #4] + str s31, [x0, #4+2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride +.endif +3: +.endm + +.macro load_n_incr_16 dst, src, incr, w +.if \w == 4 + ld1 {\dst\().4h}, [\src], \incr +.else + ld1 {\dst\().8h}, [\src], \incr +.endif +.endm + +// void dav1d_cdef_paddingX_16bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, int h, +// enum CdefEdgeFlags edges); + +.macro padding_func_16 w, stride, reg +function cdef_padding\w\()_16bpc_neon, export=1 + movi v30.8h, #0x80, lsl #8 + mov v31.16b, v30.16b + sub x0, x0, #2*(2*\stride+2) + tst w6, #4 // CDEF_HAVE_TOP + b.ne 1f + // !CDEF_HAVE_TOP + st1 {v30.8h, v31.8h}, [x0], #32 +.if \w == 8 + st1 {v30.8h, v31.8h}, [x0], #32 +.endif + b 3f +1: + // CDEF_HAVE_TOP + add x9, x4, x2 + pad_top_bot_16 x4, x9, \w, \stride, \reg, 0 + + // Middle section +3: + tst w6, #1 // CDEF_HAVE_LEFT + b.eq 2f + // CDEF_HAVE_LEFT + tst w6, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + ld1 {v0.s}[0], [x3], #4 + ldr s2, [x1, #2*\w] + load_n_incr_16 v1, x1, x2, \w + subs w5, w5, #1 + str s0, [x0] + stur \reg\()1, [x0, #4] + str s2, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 0b + b 3f +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ld1 {v0.s}[0], [x3], #4 + load_n_incr_16 v1, x1, x2, \w + subs w5, w5, #1 + str s0, [x0] + stur \reg\()1, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 1b + b 3f +2: + tst w6, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + ldr s1, [x1, #2*\w] + load_n_incr_16 v0, x1, x2, \w + subs w5, w5, #1 + str s31, [x0] + stur \reg\()0, [x0, #4] + str s1, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 0b + b 3f +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + load_n_incr_16 v0, x1, x2, \w + subs w5, w5, #1 + str s31, [x0] + stur \reg\()0, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 1b + +3: + tst w6, #8 // CDEF_HAVE_BOTTOM + b.ne 1f + // !CDEF_HAVE_BOTTOM + st1 {v30.8h, v31.8h}, [x0], #32 +.if \w == 8 + st1 {v30.8h, v31.8h}, [x0], #32 +.endif + ret +1: + // CDEF_HAVE_BOTTOM + add x9, x1, x2 + pad_top_bot_16 x1, x9, \w, \stride, \reg, 1 +endfunc +.endm + +padding_func_16 8, 16, q +padding_func_16 4, 8, d + +tables + +filter 8, 16 +filter 4, 16 + +find_dir 16 diff --git a/third_party/dav1d/src/arm/64/cdef_tmpl.S b/third_party/dav1d/src/arm/64/cdef_tmpl.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/cdef_tmpl.S @@ -0,0 +1,482 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro dir_table w, stride +const directions\w + .byte -1 * \stride + 1, -2 * \stride + 2 + .byte 0 * \stride + 1, -1 * \stride + 2 + .byte 0 * \stride + 1, 0 * \stride + 2 + .byte 0 * \stride + 1, 1 * \stride + 2 + .byte 1 * \stride + 1, 2 * \stride + 2 + .byte 1 * \stride + 0, 2 * \stride + 1 + .byte 1 * \stride + 0, 2 * \stride + 0 + .byte 1 * \stride + 0, 2 * \stride - 1 +// Repeated, to avoid & 7 + .byte -1 * \stride + 1, -2 * \stride + 2 + .byte 0 * \stride + 1, -1 * \stride + 2 + .byte 0 * \stride + 1, 0 * \stride + 2 + .byte 0 * \stride + 1, 1 * \stride + 2 + .byte 1 * \stride + 1, 2 * \stride + 2 + .byte 1 * \stride + 0, 2 * \stride + 1 +endconst +.endm + +.macro tables +dir_table 8, 16 +dir_table 4, 8 + +const pri_taps + .byte 4, 2, 3, 3 +endconst +.endm + +.macro load_px d1, d2, w +.if \w == 8 + add x6, x2, w9, sxtb #1 // x + off + sub x9, x2, w9, sxtb #1 // x - off + ld1 {\d1\().8h}, [x6] // p0 + ld1 {\d2\().8h}, [x9] // p1 +.else + add x6, x2, w9, sxtb #1 // x + off + sub x9, x2, w9, sxtb #1 // x - off + ld1 {\d1\().4h}, [x6] // p0 + add x6, x6, #2*8 // += stride + ld1 {\d2\().4h}, [x9] // p1 + add x9, x9, #2*8 // += stride + ld1 {\d1\().d}[1], [x6] // p0 + ld1 {\d2\().d}[1], [x9] // p1 +.endif +.endm +.macro handle_pixel s1, s2, thresh_vec, shift, tap, min +.if \min + umin v2.8h, v2.8h, \s1\().8h + smax v3.8h, v3.8h, \s1\().8h + umin v2.8h, v2.8h, \s2\().8h + smax v3.8h, v3.8h, \s2\().8h +.endif + uabd v16.8h, v0.8h, \s1\().8h // abs(diff) + uabd v20.8h, v0.8h, \s2\().8h // abs(diff) + ushl v17.8h, v16.8h, \shift // abs(diff) >> shift + ushl v21.8h, v20.8h, \shift // abs(diff) >> shift + uqsub v17.8h, \thresh_vec, v17.8h // clip = imax(0, threshold - (abs(diff) >> shift)) + uqsub v21.8h, \thresh_vec, v21.8h // clip = imax(0, threshold - (abs(diff) >> shift)) + sub v18.8h, \s1\().8h, v0.8h // diff = p0 - px + sub v22.8h, \s2\().8h, v0.8h // diff = p1 - px + neg v16.8h, v17.8h // -clip + neg v20.8h, v21.8h // -clip + smin v18.8h, v18.8h, v17.8h // imin(diff, clip) + smin v22.8h, v22.8h, v21.8h // imin(diff, clip) + dup v19.8h, \tap // taps[k] + smax v18.8h, v18.8h, v16.8h // constrain() = imax(imin(diff, clip), -clip) + smax v22.8h, v22.8h, v20.8h // constrain() = imax(imin(diff, clip), -clip) + mla v1.8h, v18.8h, v19.8h // sum += taps[k] * constrain() + mla v1.8h, v22.8h, v19.8h // sum += taps[k] * constrain() +.endm + +// void dav1d_cdef_filterX_Ybpc_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint16_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h, size_t edges); +.macro filter_func w, bpc, pri, sec, min, suffix +function cdef_filter\w\suffix\()_\bpc\()bpc_neon +.if \bpc == 8 + ldr w8, [sp] // bitdepth_max + cmp w8, #0xf + b.eq cdef_filter\w\suffix\()_edged_8bpc_neon +.endif +.if \pri +.if \bpc == 16 + ldr w9, [sp, #8] // bitdepth_max + clz w9, w9 + sub w9, w9, #24 // -bitdepth_min_8 + neg w9, w9 // bitdepth_min_8 +.endif + movrel x8, pri_taps +.if \bpc == 16 + lsr w9, w3, w9 // pri_strength >> bitdepth_min_8 + and w9, w9, #1 // (pri_strength >> bitdepth_min_8) & 1 +.else + and w9, w3, #1 +.endif + add x8, x8, w9, uxtw #1 +.endif + movrel x9, directions\w + add x5, x9, w5, uxtw #1 + movi v30.4h, #15 + dup v28.4h, w6 // damping + +.if \pri + dup v25.8h, w3 // threshold +.endif +.if \sec + dup v27.8h, w4 // threshold +.endif + trn1 v24.4h, v25.4h, v27.4h + clz v24.4h, v24.4h // clz(threshold) + sub v24.4h, v30.4h, v24.4h // ulog2(threshold) + uqsub v24.4h, v28.4h, v24.4h // shift = imax(0, damping - ulog2(threshold)) + neg v24.4h, v24.4h // -shift +.if \sec + dup v26.8h, v24.h[1] +.endif +.if \pri + dup v24.8h, v24.h[0] +.endif + +1: +.if \w == 8 + ld1 {v0.8h}, [x2] // px +.else + add x12, x2, #2*8 + ld1 {v0.4h}, [x2] // px + ld1 {v0.d}[1], [x12] // px +.endif + + movi v1.8h, #0 // sum +.if \min + mov v2.16b, v0.16b // min + mov v3.16b, v0.16b // max +.endif + + // Instead of loading sec_taps 2, 1 from memory, just set it + // to 2 initially and decrease for the second round. + // This is also used as loop counter. + mov w11, #2 // sec_taps[0] + +2: +.if \pri + ldrb w9, [x5] // off1 + + load_px v4, v5, \w +.endif + +.if \sec + add x5, x5, #4 // +2*2 + ldrb w9, [x5] // off2 + load_px v6, v7, \w +.endif + +.if \pri + ldrb w10, [x8] // *pri_taps + + handle_pixel v4, v5, v25.8h, v24.8h, w10, \min +.endif + +.if \sec + add x5, x5, #8 // +2*4 + ldrb w9, [x5] // off3 + load_px v4, v5, \w + + handle_pixel v6, v7, v27.8h, v26.8h, w11, \min + + handle_pixel v4, v5, v27.8h, v26.8h, w11, \min + + sub x5, x5, #11 // x5 -= 2*(2+4); x5 += 1; +.else + add x5, x5, #1 // x5 += 1 +.endif + subs w11, w11, #1 // sec_tap-- (value) +.if \pri + add x8, x8, #1 // pri_taps++ (pointer) +.endif + b.ne 2b + + sshr v4.8h, v1.8h, #15 // -(sum < 0) + add v1.8h, v1.8h, v4.8h // sum - (sum < 0) + srshr v1.8h, v1.8h, #4 // (8 + sum - (sum < 0)) >> 4 + add v0.8h, v0.8h, v1.8h // px + (8 + sum ...) >> 4 +.if \min + smin v0.8h, v0.8h, v3.8h + smax v0.8h, v0.8h, v2.8h // iclip(px + .., min, max) +.endif +.if \bpc == 8 + xtn v0.8b, v0.8h +.endif +.if \w == 8 + add x2, x2, #2*16 // tmp += tmp_stride + subs w7, w7, #1 // h-- +.if \bpc == 8 + st1 {v0.8b}, [x0], x1 +.else + st1 {v0.8h}, [x0], x1 +.endif +.else +.if \bpc == 8 + st1 {v0.s}[0], [x0], x1 +.else + st1 {v0.d}[0], [x0], x1 +.endif + add x2, x2, #2*16 // tmp += 2*tmp_stride + subs w7, w7, #2 // h -= 2 +.if \bpc == 8 + st1 {v0.s}[1], [x0], x1 +.else + st1 {v0.d}[1], [x0], x1 +.endif +.endif + + // Reset pri_taps and directions back to the original point + sub x5, x5, #2 +.if \pri + sub x8, x8, #2 +.endif + + b.gt 1b + ret +endfunc +.endm + +.macro filter w, bpc +filter_func \w, \bpc, pri=1, sec=0, min=0, suffix=_pri +filter_func \w, \bpc, pri=0, sec=1, min=0, suffix=_sec +filter_func \w, \bpc, pri=1, sec=1, min=1, suffix=_pri_sec + +function cdef_filter\w\()_\bpc\()bpc_neon, export=1 + cbnz w3, 1f // pri_strength + b cdef_filter\w\()_sec_\bpc\()bpc_neon // only sec +1: + cbnz w4, 1f // sec_strength + b cdef_filter\w\()_pri_\bpc\()bpc_neon // only pri +1: + b cdef_filter\w\()_pri_sec_\bpc\()bpc_neon // both pri and sec +endfunc +.endm + +const div_table + .short 840, 420, 280, 210, 168, 140, 120, 105 +endconst + +const alt_fact + .short 420, 210, 140, 105, 105, 105, 105, 105, 140, 210, 420, 0 +endconst + +.macro cost_alt d1, d2, s1, s2, s3, s4 + smull v22.4s, \s1\().4h, \s1\().4h // sum_alt[n]*sum_alt[n] + smull2 v23.4s, \s1\().8h, \s1\().8h + smull v24.4s, \s2\().4h, \s2\().4h + smull v25.4s, \s3\().4h, \s3\().4h // sum_alt[n]*sum_alt[n] + smull2 v26.4s, \s3\().8h, \s3\().8h + smull v27.4s, \s4\().4h, \s4\().4h + mul v22.4s, v22.4s, v29.4s // sum_alt[n]^2*fact + mla v22.4s, v23.4s, v30.4s + mla v22.4s, v24.4s, v31.4s + mul v25.4s, v25.4s, v29.4s // sum_alt[n]^2*fact + mla v25.4s, v26.4s, v30.4s + mla v25.4s, v27.4s, v31.4s + addv \d1, v22.4s // *cost_ptr + addv \d2, v25.4s // *cost_ptr +.endm + +.macro find_best s1, s2, s3 +.ifnb \s2 + mov w5, \s2\().s[0] +.endif + cmp w4, w1 // cost[n] > best_cost + csel w0, w3, w0, gt // best_dir = n + csel w1, w4, w1, gt // best_cost = cost[n] +.ifnb \s2 + add w3, w3, #1 // n++ + cmp w5, w1 // cost[n] > best_cost + mov w4, \s3\().s[0] + csel w0, w3, w0, gt // best_dir = n + csel w1, w5, w1, gt // best_cost = cost[n] + add w3, w3, #1 // n++ +.endif +.endm + +// int dav1d_cdef_find_dir_Xbpc_neon(const pixel *img, const ptrdiff_t stride, +// unsigned *const var) +.macro find_dir bpc +function cdef_find_dir_\bpc\()bpc_neon, export=1 +.if \bpc == 16 + str d8, [sp, #-0x10]! + clz w3, w3 // clz(bitdepth_max) + sub w3, w3, #24 // -bitdepth_min_8 + dup v8.8h, w3 +.endif + sub sp, sp, #32 // cost + mov w3, #8 +.if \bpc == 8 + movi v31.16b, #128 +.else + movi v31.8h, #128 +.endif + movi v30.16b, #0 + movi v1.8h, #0 // v0-v1 sum_diag[0] + movi v3.8h, #0 // v2-v3 sum_diag[1] + movi v5.8h, #0 // v4-v5 sum_hv[0-1] + movi v7.8h, #0 // v6-v7 sum_alt[0] + movi v17.8h, #0 // v16-v17 sum_alt[1] + movi v18.8h, #0 // v18-v19 sum_alt[2] + movi v19.8h, #0 + movi v21.8h, #0 // v20-v21 sum_alt[3] + +.irpc i, 01234567 +.if \bpc == 8 + ld1 {v26.8b}, [x0], x1 + usubl v26.8h, v26.8b, v31.8b +.else + ld1 {v26.8h}, [x0], x1 + ushl v26.8h, v26.8h, v8.8h + sub v26.8h, v26.8h, v31.8h +.endif + + addv h25, v26.8h // [y] + rev64 v27.8h, v26.8h + addp v28.8h, v26.8h, v30.8h // [(x >> 1)] + add v5.8h, v5.8h, v26.8h // sum_hv[1] + ext v27.16b, v27.16b, v27.16b, #8 // [-x] + rev64 v29.4h, v28.4h // [-(x >> 1)] + ins v4.h[\i], v25.h[0] // sum_hv[0] + +.if \i == 0 + mov v0.16b, v26.16b // sum_diag[0] + mov v2.16b, v27.16b // sum_diag[1] + mov v6.16b, v28.16b // sum_alt[0] + mov v16.16b, v29.16b // sum_alt[1] +.else + ext v22.16b, v30.16b, v26.16b, #(16-2*\i) + ext v23.16b, v26.16b, v30.16b, #(16-2*\i) + ext v24.16b, v30.16b, v27.16b, #(16-2*\i) + ext v25.16b, v27.16b, v30.16b, #(16-2*\i) + add v0.8h, v0.8h, v22.8h // sum_diag[0] + add v1.8h, v1.8h, v23.8h // sum_diag[0] + add v2.8h, v2.8h, v24.8h // sum_diag[1] + add v3.8h, v3.8h, v25.8h // sum_diag[1] + ext v22.16b, v30.16b, v28.16b, #(16-2*\i) + ext v23.16b, v28.16b, v30.16b, #(16-2*\i) + ext v24.16b, v30.16b, v29.16b, #(16-2*\i) + ext v25.16b, v29.16b, v30.16b, #(16-2*\i) + add v6.8h, v6.8h, v22.8h // sum_alt[0] + add v7.4h, v7.4h, v23.4h // sum_alt[0] + add v16.8h, v16.8h, v24.8h // sum_alt[1] + add v17.4h, v17.4h, v25.4h // sum_alt[1] +.endif +.if \i < 6 + ext v22.16b, v30.16b, v26.16b, #(16-2*(3-(\i/2))) + ext v23.16b, v26.16b, v30.16b, #(16-2*(3-(\i/2))) + add v18.8h, v18.8h, v22.8h // sum_alt[2] + add v19.4h, v19.4h, v23.4h // sum_alt[2] +.else + add v18.8h, v18.8h, v26.8h // sum_alt[2] +.endif +.if \i == 0 + mov v20.16b, v26.16b // sum_alt[3] +.elseif \i == 1 + add v20.8h, v20.8h, v26.8h // sum_alt[3] +.else + ext v24.16b, v30.16b, v26.16b, #(16-2*(\i/2)) + ext v25.16b, v26.16b, v30.16b, #(16-2*(\i/2)) + add v20.8h, v20.8h, v24.8h // sum_alt[3] + add v21.4h, v21.4h, v25.4h // sum_alt[3] +.endif +.endr + + movi v31.4s, #105 + + smull v26.4s, v4.4h, v4.4h // sum_hv[0]*sum_hv[0] + smlal2 v26.4s, v4.8h, v4.8h + smull v27.4s, v5.4h, v5.4h // sum_hv[1]*sum_hv[1] + smlal2 v27.4s, v5.8h, v5.8h + mul v26.4s, v26.4s, v31.4s // cost[2] *= 105 + mul v27.4s, v27.4s, v31.4s // cost[6] *= 105 + addv s4, v26.4s // cost[2] + addv s5, v27.4s // cost[6] + + rev64 v1.8h, v1.8h + rev64 v3.8h, v3.8h + ext v1.16b, v1.16b, v1.16b, #10 // sum_diag[0][14-n] + ext v3.16b, v3.16b, v3.16b, #10 // sum_diag[1][14-n] + + str s4, [sp, #2*4] // cost[2] + str s5, [sp, #6*4] // cost[6] + + movrel x4, div_table + ld1 {v31.8h}, [x4] + + smull v22.4s, v0.4h, v0.4h // sum_diag[0]*sum_diag[0] + smull2 v23.4s, v0.8h, v0.8h + smlal v22.4s, v1.4h, v1.4h + smlal2 v23.4s, v1.8h, v1.8h + smull v24.4s, v2.4h, v2.4h // sum_diag[1]*sum_diag[1] + smull2 v25.4s, v2.8h, v2.8h + smlal v24.4s, v3.4h, v3.4h + smlal2 v25.4s, v3.8h, v3.8h + uxtl v30.4s, v31.4h // div_table + uxtl2 v31.4s, v31.8h + mul v22.4s, v22.4s, v30.4s // cost[0] + mla v22.4s, v23.4s, v31.4s // cost[0] + mul v24.4s, v24.4s, v30.4s // cost[4] + mla v24.4s, v25.4s, v31.4s // cost[4] + addv s0, v22.4s // cost[0] + addv s2, v24.4s // cost[4] + + movrel x5, alt_fact + ld1 {v29.4h, v30.4h, v31.4h}, [x5]// div_table[2*m+1] + 105 + + str s0, [sp, #0*4] // cost[0] + str s2, [sp, #4*4] // cost[4] + + uxtl v29.4s, v29.4h // div_table[2*m+1] + 105 + uxtl v30.4s, v30.4h + uxtl v31.4s, v31.4h + + cost_alt s6, s16, v6, v7, v16, v17 // cost[1], cost[3] + cost_alt s18, s20, v18, v19, v20, v21 // cost[5], cost[7] + str s6, [sp, #1*4] // cost[1] + str s16, [sp, #3*4] // cost[3] + + mov w0, #0 // best_dir + mov w1, v0.s[0] // best_cost + mov w3, #1 // n + + str s18, [sp, #5*4] // cost[5] + str s20, [sp, #7*4] // cost[7] + + mov w4, v6.s[0] + + find_best v6, v4, v16 + find_best v16, v2, v18 + find_best v18, v5, v20 + find_best v20 + + eor w3, w0, #4 // best_dir ^4 + ldr w4, [sp, w3, uxtw #2] + sub w1, w1, w4 // best_cost - cost[best_dir ^ 4] + lsr w1, w1, #10 + str w1, [x2] // *var + + add sp, sp, #32 +.if \bpc == 16 + ldr d8, [sp], 0x10 +.endif + ret +endfunc +.endm diff --git a/third_party/dav1d/src/arm/64/ipred.S b/third_party/dav1d/src/arm/64/ipred.S --- a/third_party/dav1d/src/arm/64/ipred.S +++ b/third_party/dav1d/src/arm/64/ipred.S @@ -26,15 +26,15 @@ */ #include "src/arm/asm.S" #include "util.S" -// void ipred_dc_128_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_128_neon, export=1 +// void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_128_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_dc_128_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] movi v0.16b, #128 @@ -95,15 +95,15 @@ .hword L(ipred_dc_128_tbl) - 16b .hword L(ipred_dc_128_tbl) - 8b .hword L(ipred_dc_128_tbl) - 4b endfunc -// void ipred_v_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_v_neon, export=1 +// void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_v_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] add x2, x2, #1 @@ -130,11 +130,11 @@ st1 {v0.8b}, [x0], x1 st1 {v0.8b}, [x6], x1 b.gt 8b ret 160: - ld1 {v0.16b}, [x2], #16 + ld1 {v0.16b}, [x2] 16: st1 {v0.16b}, [x0], x1 st1 {v0.16b}, [x6], x1 subs w4, w4, #4 st1 {v0.16b}, [x0], x1 @@ -168,15 +168,15 @@ .hword L(ipred_v_tbl) - 160b .hword L(ipred_v_tbl) - 80b .hword L(ipred_v_tbl) - 40b endfunc -// void ipred_h_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_h_neon, export=1 +// void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_h_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] sub x2, x2, #4 @@ -249,15 +249,15 @@ .hword L(ipred_h_tbl) - 16b .hword L(ipred_h_tbl) - 8b .hword L(ipred_h_tbl) - 4b endfunc -// void ipred_dc_top_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_top_neon, export=1 +// void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_dc_top_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] add x2, x2, #1 @@ -349,15 +349,15 @@ .hword L(ipred_dc_top_tbl) - 160b .hword L(ipred_dc_top_tbl) - 80b .hword L(ipred_dc_top_tbl) - 40b endfunc -// void ipred_dc_left_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_left_neon, export=1 +// void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_8bpc_neon, export=1 sub x2, x2, w4, uxtw clz w3, w3 clz w7, w4 adr x5, L(ipred_dc_left_tbl) sub w3, w3, #20 // 25 leading bits, minus table offset 5 @@ -470,15 +470,15 @@ .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w16) .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w8) .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w4) endfunc -// void ipred_dc_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_neon, export=1 +// void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_8bpc_neon, export=1 sub x2, x2, w4, uxtw add w7, w3, w4 // width + height clz w3, w3 clz w6, w4 dup v16.8h, w7 // width + height @@ -606,22 +606,22 @@ uaddlv h1, v1.16b uaddlv h2, v2.16b cmp w4, #32 add v0.4h, v0.4h, v1.4h add v0.4h, v0.4h, v2.4h - ushl v0.4h, v0.4h, v17.4h + ushl v4.4h, v0.4h, v17.4h b.eq 1f // h = 8/16/64 cmp w4, #8 mov w16, #(0x3334/2) mov w17, #(0x5556/2) csel w16, w16, w17, eq dup v16.4h, w16 - sqdmulh v0.4h, v0.4h, v16.4h + sqdmulh v4.4h, v4.4h, v16.4h 1: - dup v0.16b, v0.b[0] - dup v1.16b, v0.b[0] + dup v0.16b, v4.b[0] + dup v1.16b, v4.b[0] 2: st1 {v0.16b, v1.16b}, [x0], x1 st1 {v0.16b, v1.16b}, [x6], x1 subs w4, w4, #4 st1 {v0.16b, v1.16b}, [x0], x1 @@ -638,14 +638,10 @@ add v0.4h, v0.4h, v1.4h add v2.4h, v2.4h, v3.4h add v0.4h, v0.4h, v2.4h br x3 L(ipred_dc_w64): - mov v1.16b, v0.16b - mov v2.16b, v0.16b - mov v3.16b, v0.16b -2: add x2, x2, #1 ld1 {v1.16b, v2.16b, v3.16b, v4.16b}, [x2] add v0.4h, v0.4h, v16.4h uaddlv h1, v1.16b uaddlv h2, v2.16b @@ -654,23 +650,23 @@ add v1.4h, v1.4h, v2.4h add v3.4h, v3.4h, v4.4h cmp w4, #64 add v0.4h, v0.4h, v1.4h add v0.4h, v0.4h, v3.4h - ushl v0.4h, v0.4h, v17.4h + ushl v4.4h, v0.4h, v17.4h b.eq 1f // h = 16/32 mov w16, #(0x5556/2) movk w16, #(0x3334/2), lsl #16 lsr w16, w16, w4 dup v16.4h, w16 - sqdmulh v0.4h, v0.4h, v16.4h + sqdmulh v4.4h, v4.4h, v16.4h 1: - dup v0.16b, v0.b[0] - dup v1.16b, v0.b[0] - dup v2.16b, v0.b[0] - dup v3.16b, v0.b[0] + dup v0.16b, v4.b[0] + dup v1.16b, v4.b[0] + dup v2.16b, v4.b[0] + dup v3.16b, v4.b[0] 2: st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 subs w4, w4, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 @@ -689,15 +685,15 @@ .hword L(ipred_dc_tbl) - L(ipred_dc_w16) .hword L(ipred_dc_tbl) - L(ipred_dc_w8) .hword L(ipred_dc_tbl) - L(ipred_dc_w4) endfunc -// void ipred_paeth_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_paeth_neon, export=1 +// void ipred_paeth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_8bpc_neon, export=1 clz w9, w3 adr x5, L(ipred_paeth_tbl) sub w9, w9, #25 ldrh w9, [x5, w9, uxtw #1] ld1r {v4.16b}, [x2] @@ -866,15 +862,15 @@ .hword L(ipred_paeth_tbl) - 160b .hword L(ipred_paeth_tbl) - 80b .hword L(ipred_paeth_tbl) - 40b endfunc -// void ipred_smooth_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_smooth_neon, export=1 +// void ipred_smooth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_8bpc_neon, export=1 movrel x10, X(sm_weights) add x11, x10, w4, uxtw add x10, x10, w3, uxtw clz w9, w3 adr x5, L(ipred_smooth_tbl) @@ -1044,15 +1040,15 @@ .hword L(ipred_smooth_tbl) - 160b .hword L(ipred_smooth_tbl) - 80b .hword L(ipred_smooth_tbl) - 40b endfunc -// void ipred_smooth_v_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_smooth_v_neon, export=1 +// void ipred_smooth_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_8bpc_neon, export=1 movrel x7, X(sm_weights) add x7, x7, w4, uxtw clz w9, w3 adr x5, L(ipred_smooth_v_tbl) sub x8, x2, w4, uxtw @@ -1182,15 +1178,15 @@ .hword L(ipred_smooth_v_tbl) - 160b .hword L(ipred_smooth_v_tbl) - 80b .hword L(ipred_smooth_v_tbl) - 40b endfunc -// void ipred_smooth_h_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_smooth_h_neon, export=1 +// void ipred_smooth_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_8bpc_neon, export=1 movrel x8, X(sm_weights) add x8, x8, w3, uxtw clz w9, w3 adr x5, L(ipred_smooth_h_tbl) add x12, x2, w3, uxtw @@ -1325,15 +1321,15 @@ .hword L(ipred_smooth_h_tbl) - 160b .hword L(ipred_smooth_h_tbl) - 80b .hword L(ipred_smooth_h_tbl) - 40b endfunc -// void ipred_filter_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int filt_idx, -// const int max_width, const int max_height); -function ipred_filter_neon, export=1 +// void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height); +function ipred_filter_8bpc_neon, export=1 and w5, w5, #511 movrel x6, X(filter_intra_taps) lsl w5, w5, #6 add x6, x6, w5, uxtw ld1 {v16.8b, v17.8b, v18.8b, v19.8b}, [x6], #32 @@ -1485,14 +1481,14 @@ .hword L(ipred_filter_tbl) - 160b .hword L(ipred_filter_tbl) - 80b .hword L(ipred_filter_tbl) - 40b endfunc -// void pal_pred_neon(pixel *dst, const ptrdiff_t stride, -// const uint16_t *const pal, const uint8_t *idx, -// const int w, const int h); -function pal_pred_neon, export=1 +// void pal_pred_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_8bpc_neon, export=1 ld1 {v0.8h}, [x2] clz w9, w4 adr x6, L(pal_pred_tbl) sub w9, w9, #25 ldrh w9, [x6, w9, uxtw #1] @@ -1576,15 +1572,15 @@ .hword L(pal_pred_tbl) - 16b .hword L(pal_pred_tbl) - 8b .hword L(pal_pred_tbl) - 4b endfunc -// void ipred_cfl_128_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_128_neon, export=1 +// void ipred_cfl_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_128_8bpc_neon, export=1 clz w9, w3 adr x7, L(ipred_cfl_128_tbl) sub w9, w9, #26 ldrh w9, [x7, w9, uxtw #1] movi v0.8h, #128 // dc @@ -1697,15 +1693,15 @@ .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w8) .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w4) endfunc -// void ipred_cfl_top_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_top_neon, export=1 +// void ipred_cfl_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_top_8bpc_neon, export=1 clz w9, w3 adr x7, L(ipred_cfl_top_tbl) sub w9, w9, #26 ldrh w9, [x7, w9, uxtw #1] dup v1.8h, w6 // alpha @@ -1715,46 +1711,46 @@ lsl x1, x1, #1 br x7 4: ld1r {v0.2s}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] b L(ipred_cfl_splat_w4) 8: ld1 {v0.8b}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] b L(ipred_cfl_splat_w8) 16: ld1 {v0.16b}, [x2] uaddlv h0, v0.16b - urshr v0.8h, v0.8h, #4 + urshr v0.4h, v0.4h, #4 dup v0.8h, v0.h[0] b L(ipred_cfl_splat_w16) 32: ld1 {v2.16b, v3.16b}, [x2] uaddlv h2, v2.16b uaddlv h3, v3.16b add v2.4h, v2.4h, v3.4h - urshr v2.8h, v2.8h, #5 + urshr v2.4h, v2.4h, #5 dup v0.8h, v2.h[0] b L(ipred_cfl_splat_w16) L(ipred_cfl_top_tbl): .hword L(ipred_cfl_top_tbl) - 32b .hword L(ipred_cfl_top_tbl) - 16b .hword L(ipred_cfl_top_tbl) - 8b .hword L(ipred_cfl_top_tbl) - 4b endfunc -// void ipred_cfl_left_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_left_neon, export=1 +// void ipred_cfl_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_left_8bpc_neon, export=1 sub x2, x2, w4, uxtw clz w9, w3 clz w8, w4 adr x10, L(ipred_cfl_splat_tbl) adr x7, L(ipred_cfl_left_tbl) @@ -1770,49 +1766,49 @@ br x7 L(ipred_cfl_left_h4): ld1r {v0.2s}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] br x9 L(ipred_cfl_left_h8): ld1 {v0.8b}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] br x9 L(ipred_cfl_left_h16): ld1 {v0.16b}, [x2] uaddlv h0, v0.16b - urshr v0.8h, v0.8h, #4 + urshr v0.4h, v0.4h, #4 dup v0.8h, v0.h[0] br x9 L(ipred_cfl_left_h32): ld1 {v2.16b, v3.16b}, [x2] uaddlv h2, v2.16b uaddlv h3, v3.16b add v2.4h, v2.4h, v3.4h - urshr v2.8h, v2.8h, #5 + urshr v2.4h, v2.4h, #5 dup v0.8h, v2.h[0] br x9 L(ipred_cfl_left_tbl): .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h32) .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h16) .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h8) .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h4) endfunc -// void ipred_cfl_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_neon, export=1 +// void ipred_cfl_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_8bpc_neon, export=1 sub x2, x2, w4, uxtw add w8, w3, w4 // width + height dup v1.8h, w6 // alpha clz w9, w3 clz w6, w4 @@ -1944,34 +1940,34 @@ .hword L(ipred_cfl_tbl) - L(ipred_cfl_w16) .hword L(ipred_cfl_tbl) - L(ipred_cfl_w8) .hword L(ipred_cfl_tbl) - L(ipred_cfl_w4) endfunc -// void cfl_ac_420_neon(int16_t *const ac, const pixel *const ypx, -// const ptrdiff_t stride, const int w_pad, -// const int h_pad, const int cw, const int ch); -function ipred_cfl_ac_420_neon, export=1 +// void cfl_ac_420_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_8bpc_neon, export=1 clz w8, w5 lsl w4, w4, #2 adr x7, L(ipred_cfl_ac_420_tbl) sub w8, w8, #27 ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 sub x7, x7, w8, uxtw sub w8, w6, w4 // height - h_pad rbit w9, w5 // rbit(width) rbit w10, w6 // rbit(height) clz w9, w9 // ctz(width) clz w10, w10 // ctz(height) add w9, w9, w10 // log2sz - movi v16.4s, #1 add x10, x1, x2 + dup v31.4s, w9 lsl x2, x2, #1 - dup v17.4s, w9 - sshl v16.4s, v16.4s, v17.4s // 1 << log2sz - neg v17.4s, v17.4s // -log2sz - ushr v16.4s, v16.4s, #1 // 1 << (log2sz - 1) - mov w9, w6 + neg v31.4s, v31.4s // -log2sz br x7 L(ipred_cfl_ac_420_w4): 1: // Copy and subsample input ld1 {v0.8b}, [x1], x2 @@ -1982,41 +1978,32 @@ uaddlp v1.8h, v1.16b add v0.8h, v0.8h, v1.8h shl v0.8h, v0.8h, #1 subs w8, w8, #2 st1 {v0.8h}, [x0], #16 + add v16.8h, v16.8h, v0.8h b.gt 1b trn2 v1.2d, v0.2d, v0.2d trn2 v0.2d, v0.2d, v0.2d L(ipred_cfl_ac_420_w4_hpad): cbz w4, 3f 2: // Vertical padding (h_pad > 0) subs w4, w4, #4 st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h b.gt 2b 3: + // Aggregate the sums + add v0.8h, v16.8h, v17.8h + uaddlv s0, v0.8h // sum sub x0, x0, w6, uxtw #3 - // Sum the produced ac values - subs w6, w6, #4 - ld1 {v0.8h, v1.8h}, [x0], #32 - b.le 5f -4: - ld1 {v2.8h, v3.8h}, [x0], #32 - subs w6, w6, #4 - add v0.8h, v0.8h, v2.8h - add v1.8h, v1.8h, v3.8h - b.gt 4b -5: - add v0.8h, v0.8h, v1.8h - uaddlv s0, v0.8h // sum - sub x0, x0, w9, uxtw #3 - add v0.2s, v0.2s, v16.2s // sum += 1 << (log2sz - 1) - ushl v4.2s, v0.2s, v17.2s // sum >>= log2sz + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz dup v4.8h, v4.h[0] 6: // Subtract dc from ac ld1 {v0.8h, v1.8h}, [x0] - subs w9, w9, #4 + subs w6, w6, #4 sub v0.8h, v0.8h, v4.8h sub v1.8h, v1.8h, v4.8h st1 {v0.8h, v1.8h}, [x0], #32 b.gt 6b ret @@ -2036,10 +2023,12 @@ add v2.8h, v2.8h, v3.8h shl v0.8h, v0.8h, #1 shl v1.8h, v2.8h, #1 subs w8, w8, #2 st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h b.gt 1b mov v0.16b, v1.16b b L(ipred_cfl_ac_420_w8_hpad) L(ipred_cfl_ac_420_w8_wpad): @@ -2055,51 +2044,45 @@ dup v1.4h, v0.h[3] dup v3.4h, v0.h[7] trn2 v2.2d, v0.2d, v0.2d subs w8, w8, #2 st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [x0], #32 + add v16.4h, v16.4h, v0.4h + add v17.4h, v17.4h, v1.4h + add v18.4h, v18.4h, v2.4h + add v19.4h, v19.4h, v3.4h b.gt 1b trn1 v0.2d, v2.2d, v3.2d trn1 v1.2d, v2.2d, v3.2d L(ipred_cfl_ac_420_w8_hpad): cbz w4, 3f 2: // Vertical padding (h_pad > 0) subs w4, w4, #4 st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h st1 {v0.8h, v1.8h}, [x0], #32 + add v18.8h, v18.8h, v0.8h + add v19.8h, v19.8h, v1.8h b.gt 2b 3: L(ipred_cfl_ac_420_w8_calc_subtract_dc): - sub x0, x0, w6, uxtw #4 - // Sum the produced ac values - subs w6, w6, #4 - ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 - b.le 5f -4: - ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 - subs w6, w6, #4 - add v0.8h, v0.8h, v4.8h - add v1.8h, v1.8h, v5.8h - add v2.8h, v2.8h, v6.8h - add v3.8h, v3.8h, v7.8h - b.gt 4b -5: - add v0.8h, v0.8h, v1.8h - add v2.8h, v2.8h, v3.8h + // Aggregate the sums + add v0.8h, v16.8h, v17.8h + add v2.8h, v18.8h, v19.8h uaddlp v0.4s, v0.8h uaddlp v2.4s, v2.8h add v0.4s, v0.4s, v2.4s addv s0, v0.4s // sum - sub x0, x0, w9, uxtw #4 - add v0.2s, v0.2s, v16.2s // sum += 1 << (log2sz - 1) - ushl v4.2s, v0.2s, v17.2s // sum >>= log2sz + sub x0, x0, w6, uxtw #4 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz dup v4.8h, v4.h[0] 6: // Subtract dc from ac ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] - subs w9, w9, #4 + subs w6, w6, #4 sub v0.8h, v0.8h, v4.8h sub v1.8h, v1.8h, v4.8h sub v2.8h, v2.8h, v4.8h sub v3.8h, v3.8h, v4.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 @@ -2134,10 +2117,14 @@ shl v1.8h, v1.8h, #1 shl v2.8h, v4.8h, #1 shl v3.8h, v5.8h, #1 subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2171,10 +2158,14 @@ dup v5.4h, v3.h[3] trn1 v1.2d, v1.2d, v4.2d trn1 v3.2d, v3.2d, v5.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2194,10 +2185,14 @@ shl v2.8h, v4.8h, #1 dup v1.8h, v0.h[7] dup v3.8h, v2.h[7] subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2219,21 +2214,33 @@ dup v3.8h, v2.h[3] trn1 v0.2d, v0.2d, v1.2d trn1 v2.2d, v2.2d, v3.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_420_w16_hpad): cbz w4, 3f 2: // Vertical padding (h_pad > 0) subs w4, w4, #4 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 2b 3: // Double the height and reuse the w8 summing/subtracting lsl w6, w6, #1 @@ -2251,34 +2258,34 @@ .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad1) .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad2) .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad3) endfunc -// void cfl_ac_422_neon(int16_t *const ac, const pixel *const ypx, -// const ptrdiff_t stride, const int w_pad, -// const int h_pad, const int cw, const int ch); -function ipred_cfl_ac_422_neon, export=1 +// void cfl_ac_422_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_8bpc_neon, export=1 clz w8, w5 lsl w4, w4, #2 adr x7, L(ipred_cfl_ac_422_tbl) sub w8, w8, #27 ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 sub x7, x7, w8, uxtw sub w8, w6, w4 // height - h_pad rbit w9, w5 // rbit(width) rbit w10, w6 // rbit(height) clz w9, w9 // ctz(width) clz w10, w10 // ctz(height) add w9, w9, w10 // log2sz - movi v16.4s, #1 add x10, x1, x2 + dup v31.4s, w9 lsl x2, x2, #1 - dup v17.4s, w9 - sshl v16.4s, v16.4s, v17.4s // 1 << log2sz - neg v17.4s, v17.4s // -log2sz - ushr v16.4s, v16.4s, #1 // 1 << (log2sz - 1) - mov w9, w6 + neg v31.4s, v31.4s // -log2sz br x7 L(ipred_cfl_ac_422_w4): 1: // Copy and subsample input ld1 {v0.8b}, [x1], x2 @@ -2288,10 +2295,12 @@ uaddlp v0.8h, v0.16b uaddlp v1.8h, v1.16b shl v0.8h, v0.8h, #2 shl v1.8h, v1.8h, #2 subs w8, w8, #4 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h st1 {v0.8h, v1.8h}, [x0], #32 b.gt 1b trn2 v0.2d, v1.2d, v1.2d trn2 v1.2d, v1.2d, v1.2d b L(ipred_cfl_ac_420_w4_hpad) @@ -2311,10 +2320,14 @@ shl v1.8h, v1.8h, #2 shl v2.8h, v2.8h, #2 shl v3.8h, v3.8h, #2 subs w8, w8, #4 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v3.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w8_hpad) @@ -2336,10 +2349,14 @@ trn1 v0.2d, v0.2d, v4.2d trn2 v3.2d, v2.2d, v7.2d trn1 v2.2d, v2.2d, v6.2d subs w8, w8, #4 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v3.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w8_hpad) @@ -2361,10 +2378,14 @@ shl v1.8h, v1.8h, #2 shl v2.8h, v2.8h, #2 shl v3.8h, v3.8h, #2 subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2386,10 +2407,14 @@ dup v5.4h, v3.h[3] trn1 v1.2d, v1.2d, v4.2d trn1 v3.2d, v3.2d, v5.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2403,10 +2428,14 @@ shl v2.8h, v2.8h, #2 dup v1.8h, v0.h[7] dup v3.8h, v2.h[7] subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2422,10 +2451,14 @@ dup v3.8h, v2.h[3] trn1 v0.2d, v0.2d, v1.2d trn1 v2.2d, v2.2d, v3.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) diff --git a/third_party/dav1d/src/arm/64/ipred16.S b/third_party/dav1d/src/arm/64/ipred16.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/ipred16.S @@ -0,0 +1,2834 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void ipred_dc_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height, +// const int bitdepth_max); +function ipred_dc_128_16bpc_neon, export=1 + ldr w8, [sp] + clz w3, w3 + adr x5, L(ipred_dc_128_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + dup v0.8h, w8 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + urshr v0.8h, v0.8h, #1 + br x5 +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + mov v1.16b, v0.16b +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + sub x1, x1, #64 +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_128_tbl): + .hword L(ipred_dc_128_tbl) - 640b + .hword L(ipred_dc_128_tbl) - 320b + .hword L(ipred_dc_128_tbl) - 160b + .hword L(ipred_dc_128_tbl) - 8b + .hword L(ipred_dc_128_tbl) - 4b +endfunc + +// void ipred_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_v_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #2 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1 {v0.4h}, [x2] +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +80: + ld1 {v0.8h}, [x2] +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + ld1 {v0.8h, v1.8h}, [x2] +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + sub x1, x1, #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_v_tbl): + .hword L(ipred_v_tbl) - 640b + .hword L(ipred_v_tbl) - 320b + .hword L(ipred_v_tbl) - 160b + .hword L(ipred_v_tbl) - 80b + .hword L(ipred_v_tbl) - 40b +endfunc + +// void ipred_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_h_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + sub x2, x2, #8 + sub x5, x5, w3, uxtw + mov x7, #-8 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +4: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + st1 {v3.4h}, [x0], x1 + st1 {v2.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v1.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +16: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 16b + ret +32: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + stp q3, q3, [x0, #32] + stp q2, q2, [x6, #32] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + stp q1, q1, [x0, #32] + stp q0, q0, [x6, #32] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 32b + ret +64: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + stp q3, q3, [x0, #32] + stp q2, q2, [x6, #32] + stp q3, q3, [x0, #64] + stp q2, q2, [x6, #64] + stp q3, q3, [x0, #96] + stp q2, q2, [x6, #96] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + stp q1, q1, [x0, #32] + stp q0, q0, [x6, #32] + stp q1, q1, [x0, #64] + stp q0, q0, [x6, #64] + stp q1, q1, [x0, #96] + stp q0, q0, [x6, #96] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_h_tbl): + .hword L(ipred_h_tbl) - 64b + .hword L(ipred_h_tbl) - 32b + .hword L(ipred_h_tbl) - 16b + .hword L(ipred_h_tbl) - 8b + .hword L(ipred_h_tbl) - 4b +endfunc + +// void ipred_dc_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_dc_top_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #2 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.4h, v0.h[0] +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +80: + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + ld1 {v0.8h, v1.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addv h0, v0.8h + urshr v2.4h, v0.4h, #4 + dup v0.8h, v2.h[0] + dup v1.8h, v2.h[0] +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #5 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #6 + sub x1, x1, #64 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_top_tbl): + .hword L(ipred_dc_top_tbl) - 640b + .hword L(ipred_dc_top_tbl) - 320b + .hword L(ipred_dc_top_tbl) - 160b + .hword L(ipred_dc_top_tbl) - 80b + .hword L(ipred_dc_top_tbl) - 40b +endfunc + +// void ipred_dc_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_16bpc_neon, export=1 + sub x2, x2, w4, uxtw #1 + clz w3, w3 + clz w7, w4 + adr x5, L(ipred_dc_left_tbl) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w7, w7, #25 + ldrh w3, [x5, w3, uxtw #1] + ldrh w7, [x5, w7, uxtw #1] + sub x3, x5, w3, uxtw + sub x5, x5, w7, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_left_h4): + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + br x3 +L(ipred_dc_left_w4): + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt L(ipred_dc_left_w4) + ret + +L(ipred_dc_left_h8): + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x3 +L(ipred_dc_left_w8): + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt L(ipred_dc_left_w8) + ret + +L(ipred_dc_left_h16): + ld1 {v0.8h, v1.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addv h0, v0.8h + urshr v2.4h, v0.4h, #4 + dup v0.8h, v2.h[0] + dup v1.8h, v2.h[0] + br x3 +L(ipred_dc_left_w16): + mov v1.16b, v0.16b +1: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_h32): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlp v0.4s, v0.8h + addv s0, v0.4s + rshrn v4.4h, v0.4s, #5 + dup v0.8h, v4.h[0] + br x3 +L(ipred_dc_left_w32): + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b +1: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_h64): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #6 + dup v0.8h, v4.h[0] + br x3 +L(ipred_dc_left_w64): + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + sub x1, x1, #64 +1: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_tbl): + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h4) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w4) +endfunc + +// void ipred_dc_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_16bpc_neon, export=1 + sub x2, x2, w4, uxtw #1 + add w7, w3, w4 // width + height + clz w3, w3 + clz w6, w4 + dup v16.4s, w7 // width + height + adr x5, L(ipred_dc_tbl) + rbit w7, w7 // rbit(width + height) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w6, w6, #25 + clz w7, w7 // ctz(width + height) + ldrh w3, [x5, w3, uxtw #1] + ldrh w6, [x5, w6, uxtw #1] + neg w7, w7 // -ctz(width + height) + sub x3, x5, w3, uxtw + sub x5, x5, w6, uxtw + ushr v16.4s, v16.4s, #1 // (width + height) >> 1 + dup v17.4s, w7 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_h4): + ld1 {v0.4h}, [x2], #8 + uaddlv s0, v0.4h + br x3 +L(ipred_dc_w4): + add x2, x2, #2 + ld1 {v1.4h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s1, v1.4h + cmp w4, #4 + add v0.2s, v0.2s, v1.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.4h, v0.h[0] +2: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h8): + ld1 {v0.8h}, [x2], #16 + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w8): + add x2, x2, #2 + ld1 {v1.8h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s1, v1.8h + cmp w4, #8 + add v0.2s, v0.2s, v1.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] +2: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h16): + ld1 {v0.8h, v1.8h}, [x2], #32 + addp v0.8h, v0.8h, v1.8h + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w16): + add x2, x2, #2 + ld1 {v1.8h, v2.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + uaddlv s1, v1.8h + cmp w4, #16 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/8/32/64 + tst w4, #(32+16+8) // 16 added to make a consecutive bitmask + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h32): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w32): + add x2, x2, #2 + ld1 {v1.8h, v2.8h, v3.8h, v4.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + addp v3.8h, v3.8h, v4.8h + addp v1.8h, v1.8h, v3.8h + uaddlv s1, v1.8h + cmp w4, #32 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16/64 + cmp w4, #8 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h64): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w64): + add x2, x2, #2 + ld1 {v1.8h, v2.8h, v3.8h, v4.8h}, [x2], #64 + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2] + addp v3.8h, v3.8h, v4.8h + addp v20.8h, v20.8h, v21.8h + addp v22.8h, v22.8h, v23.8h + addp v1.8h, v1.8h, v3.8h + addp v20.8h, v20.8h, v22.8h + addp v1.8h, v1.8h, v20.8h + uaddlv s1, v1.8h + cmp w4, #64 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 16/32 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + sub x1, x1, #64 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_tbl): + .hword L(ipred_dc_tbl) - L(ipred_dc_h64) + .hword L(ipred_dc_tbl) - L(ipred_dc_h32) + .hword L(ipred_dc_tbl) - L(ipred_dc_h16) + .hword L(ipred_dc_tbl) - L(ipred_dc_h8) + .hword L(ipred_dc_tbl) - L(ipred_dc_h4) + .hword L(ipred_dc_tbl) - L(ipred_dc_w64) + .hword L(ipred_dc_tbl) - L(ipred_dc_w32) + .hword L(ipred_dc_tbl) - L(ipred_dc_w16) + .hword L(ipred_dc_tbl) - L(ipred_dc_w8) + .hword L(ipred_dc_tbl) - L(ipred_dc_w4) +endfunc + +// void ipred_paeth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_16bpc_neon, export=1 + clz w9, w3 + adr x5, L(ipred_paeth_tbl) + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x2] + add x8, x2, #2 + sub x2, x2, #8 + sub x5, x5, w9, uxtw + mov x7, #-8 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1r {v5.2d}, [x8] + sub v6.8h, v5.8h, v4.8h // top - topleft +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 + zip1 v0.2d, v0.2d, v1.2d + zip1 v2.2d, v2.2d, v3.2d + add v16.8h, v6.8h, v0.8h // base + add v17.8h, v6.8h, v2.8h + sabd v20.8h, v5.8h, v16.8h // tdiff + sabd v21.8h, v5.8h, v17.8h + sabd v22.8h, v4.8h, v16.8h // tldiff + sabd v23.8h, v4.8h, v17.8h + sabd v16.8h, v0.8h, v16.8h // ldiff + sabd v17.8h, v2.8h, v17.8h + umin v18.8h, v20.8h, v22.8h // min(tdiff, tldiff) + umin v19.8h, v21.8h, v23.8h + cmge v20.8h, v22.8h, v20.8h // tldiff >= tdiff + cmge v21.8h, v23.8h, v21.8h + cmge v16.8h, v18.8h, v16.8h // min(tdiff, tldiff) >= ldiff + cmge v17.8h, v19.8h, v17.8h + bsl v21.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v20.16b, v5.16b, v4.16b + bit v21.16b, v2.16b, v17.16b // ldiff <= min ? left : ... + bit v20.16b, v0.16b, v16.16b + st1 {v21.d}[1], [x0], x1 + st1 {v21.d}[0], [x6], x1 + subs w4, w4, #4 + st1 {v20.d}[1], [x0], x1 + st1 {v20.d}[0], [x6], x1 + b.gt 4b + ret +80: +160: +320: +640: + ld1 {v5.8h}, [x8], #16 + mov w9, w3 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 +1: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 +2: + sub v6.8h, v5.8h, v4.8h // top - topleft + add v16.8h, v6.8h, v0.8h // base + add v17.8h, v6.8h, v1.8h + add v18.8h, v6.8h, v2.8h + add v19.8h, v6.8h, v3.8h + sabd v20.8h, v5.8h, v16.8h // tdiff + sabd v21.8h, v5.8h, v17.8h + sabd v22.8h, v5.8h, v18.8h + sabd v23.8h, v5.8h, v19.8h + sabd v24.8h, v4.8h, v16.8h // tldiff + sabd v25.8h, v4.8h, v17.8h + sabd v26.8h, v4.8h, v18.8h + sabd v27.8h, v4.8h, v19.8h + sabd v16.8h, v0.8h, v16.8h // ldiff + sabd v17.8h, v1.8h, v17.8h + sabd v18.8h, v2.8h, v18.8h + sabd v19.8h, v3.8h, v19.8h + umin v28.8h, v20.8h, v24.8h // min(tdiff, tldiff) + umin v29.8h, v21.8h, v25.8h + umin v30.8h, v22.8h, v26.8h + umin v31.8h, v23.8h, v27.8h + cmge v20.8h, v24.8h, v20.8h // tldiff >= tdiff + cmge v21.8h, v25.8h, v21.8h + cmge v22.8h, v26.8h, v22.8h + cmge v23.8h, v27.8h, v23.8h + cmge v16.8h, v28.8h, v16.8h // min(tdiff, tldiff) >= ldiff + cmge v17.8h, v29.8h, v17.8h + cmge v18.8h, v30.8h, v18.8h + cmge v19.8h, v31.8h, v19.8h + bsl v23.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v22.16b, v5.16b, v4.16b + bsl v21.16b, v5.16b, v4.16b + bsl v20.16b, v5.16b, v4.16b + bit v23.16b, v3.16b, v19.16b // ldiff <= min ? left : ... + bit v22.16b, v2.16b, v18.16b + bit v21.16b, v1.16b, v17.16b + bit v20.16b, v0.16b, v16.16b + st1 {v23.8h}, [x0], #16 + st1 {v22.8h}, [x6], #16 + subs w3, w3, #8 + st1 {v21.8h}, [x5], #16 + st1 {v20.8h}, [x10], #16 + b.le 8f + ld1 {v5.8h}, [x8], #16 + b 2b +8: + subs w4, w4, #4 + b.le 9f + // End of horizontal loop, move pointers to next four rows + sub x8, x8, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + // Load the top row as early as possible + ld1 {v5.8h}, [x8], #16 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_paeth_tbl): + .hword L(ipred_paeth_tbl) - 640b + .hword L(ipred_paeth_tbl) - 320b + .hword L(ipred_paeth_tbl) - 160b + .hword L(ipred_paeth_tbl) - 80b + .hword L(ipred_paeth_tbl) - 40b +endfunc + +// void ipred_smooth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_16bpc_neon, export=1 + movrel x10, X(sm_weights) + add x11, x10, w4, uxtw + add x10, x10, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_tbl) + sub x12, x2, w4, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x12] // bottom + add x8, x2, #2 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + sub x2, x2, #8 + mov x7, #-8 + ld1r {v6.2d}, [x8] // top + ld1r {v7.2s}, [x10] // weights_hor + dup v5.8h, v6.h[3] // right + sub v6.8h, v6.8h, v4.8h // top-bottom + uxtl v7.8h, v7.8b // weights_hor + add v31.4h, v4.4h, v5.4h // bottom+right +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + zip1 v1.2d, v1.2d, v0.2d // left, flipped + zip1 v0.2d, v3.2d, v2.2d + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v18.8h, v18.8b + smlal v20.4s, v0.4h, v7.4h // += (left-right)*weights_hor + smlal2 v21.4s, v0.8h, v7.8h + smlal v22.4s, v1.4h, v7.4h + smlal2 v23.4s, v1.8h, v7.8h + smlal v20.4s, v6.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v6.8h, v16.8h + smlal v22.4s, v6.4h, v18.4h + smlal2 v23.4s, v6.8h, v18.8h + rshrn v20.4h, v20.4s, #9 + rshrn v21.4h, v21.4s, #9 + rshrn v22.4h, v22.4s, #9 + rshrn v23.4h, v23.4s, #9 + st1 {v20.4h}, [x0], x1 + st1 {v21.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.4h}, [x0], x1 + st1 {v23.4h}, [x6], x1 + b.gt 4b + ret +80: + sub x2, x2, #8 + mov x7, #-8 + ld1 {v6.8h}, [x8] // top + ld1 {v7.8b}, [x10] // weights_hor + dup v5.8h, v6.h[7] // right + sub v6.8h, v6.8h, v4.8h // top-bottom + uxtl v7.8h, v7.8b // weights_hor + add v31.4h, v4.4h, v5.4h // bottom+right +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + ushll v24.4s, v31.4h, #8 + ushll v25.4s, v31.4h, #8 + ushll v26.4s, v31.4h, #8 + ushll v27.4s, v31.4h, #8 + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sub v2.8h, v2.8h, v5.8h + sub v3.8h, v3.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v19.8h, v19.8b + smlal v20.4s, v3.4h, v7.4h // += (left-right)*weights_hor + smlal2 v21.4s, v3.8h, v7.8h // (left flipped) + smlal v22.4s, v2.4h, v7.4h + smlal2 v23.4s, v2.8h, v7.8h + smlal v24.4s, v1.4h, v7.4h + smlal2 v25.4s, v1.8h, v7.8h + smlal v26.4s, v0.4h, v7.4h + smlal2 v27.4s, v0.8h, v7.8h + smlal v20.4s, v6.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v6.8h, v16.8h + smlal v22.4s, v6.4h, v17.4h + smlal2 v23.4s, v6.8h, v17.8h + smlal v24.4s, v6.4h, v18.4h + smlal2 v25.4s, v6.8h, v18.8h + smlal v26.4s, v6.4h, v19.4h + smlal2 v27.4s, v6.8h, v19.8h + rshrn v20.4h, v20.4s, #9 + rshrn2 v20.8h, v21.4s, #9 + rshrn v21.4h, v22.4s, #9 + rshrn2 v21.8h, v23.4s, #9 + rshrn v22.4h, v24.4s, #9 + rshrn2 v22.8h, v25.4s, #9 + rshrn v23.4h, v26.4s, #9 + rshrn2 v23.8h, v27.4s, #9 + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + add x12, x2, w3, uxtw #1 + sub x1, x1, w3, uxtw #1 + ld1r {v5.8h}, [x12] // right + sub x2, x2, #4 + mov x7, #-4 + mov w9, w3 + add v31.4h, v4.4h, v5.4h // bottom+right + +1: + ld2r {v0.8h, v1.8h}, [x2], x7 // left + ld2r {v16.8b, v17.8b}, [x11], #2 // weights_ver + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b +2: + ld1 {v7.16b}, [x10], #16 // weights_hor + ld1 {v2.8h, v3.8h}, [x8], #32 // top + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + ushll v24.4s, v31.4h, #8 + ushll v25.4s, v31.4h, #8 + ushll v26.4s, v31.4h, #8 + ushll v27.4s, v31.4h, #8 + uxtl v6.8h, v7.8b // weights_hor + uxtl2 v7.8h, v7.16b + sub v2.8h, v2.8h, v4.8h // top-bottom + sub v3.8h, v3.8h, v4.8h + smlal v20.4s, v1.4h, v6.4h // += (left-right)*weights_hor + smlal2 v21.4s, v1.8h, v6.8h // (left flipped) + smlal v22.4s, v1.4h, v7.4h + smlal2 v23.4s, v1.8h, v7.8h + smlal v24.4s, v0.4h, v6.4h + smlal2 v25.4s, v0.8h, v6.8h + smlal v26.4s, v0.4h, v7.4h + smlal2 v27.4s, v0.8h, v7.8h + smlal v20.4s, v2.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v2.8h, v16.8h + smlal v22.4s, v3.4h, v16.4h + smlal2 v23.4s, v3.8h, v16.8h + smlal v24.4s, v2.4h, v17.4h + smlal2 v25.4s, v2.8h, v17.8h + smlal v26.4s, v3.4h, v17.4h + smlal2 v27.4s, v3.8h, v17.8h + rshrn v20.4h, v20.4s, #9 + rshrn2 v20.8h, v21.4s, #9 + rshrn v21.4h, v22.4s, #9 + rshrn2 v21.8h, v23.4s, #9 + rshrn v22.4h, v24.4s, #9 + rshrn2 v22.8h, v25.4s, #9 + rshrn v23.4h, v26.4s, #9 + rshrn2 v23.8h, v27.4s, #9 + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + b.gt 2b + subs w4, w4, #2 + b.le 9f + sub x8, x8, w9, uxtw #1 + sub x10, x10, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_tbl): + .hword L(ipred_smooth_tbl) - 640b + .hword L(ipred_smooth_tbl) - 320b + .hword L(ipred_smooth_tbl) - 160b + .hword L(ipred_smooth_tbl) - 80b + .hword L(ipred_smooth_tbl) - 40b +endfunc + +// void ipred_smooth_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_16bpc_neon, export=1 + movrel x7, X(sm_weights) + add x7, x7, w4, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_v_tbl) + sub x8, x2, w4, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x8] // bottom + add x2, x2, #2 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1r {v6.2d}, [x2] // top + sub v6.8h, v6.8h, v4.8h // top-bottom +4: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v18.8h, v18.8b, #7 + sqrdmulh v20.8h, v6.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v6.8h, v18.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + st1 {v20.d}[0], [x0], x1 + st1 {v20.d}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.d}[0], [x0], x1 + st1 {v21.d}[1], [x6], x1 + b.gt 4b + ret +80: + ld1 {v6.8h}, [x2] // top + sub v6.8h, v6.8h, v4.8h // top-bottom +8: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v17.8h, v17.8b, #7 + ushll v18.8h, v18.8b, #7 + ushll v19.8h, v19.8b, #7 + sqrdmulh v20.8h, v6.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v6.8h, v17.8h + sqrdmulh v22.8h, v6.8h, v18.8h + sqrdmulh v23.8h, v6.8h, v19.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + add v22.8h, v22.8h, v4.8h + add v23.8h, v23.8h, v4.8h + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + // Set up pointers for four rows in parallel; x0, x6, x5, x8 + add x5, x0, x1 + add x8, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v17.8h, v17.8b, #7 + ushll v18.8h, v18.8b, #7 + ushll v19.8h, v19.8b, #7 +2: + ld1 {v2.8h, v3.8h}, [x2], #32 // top + sub v2.8h, v2.8h, v4.8h // top-bottom + sub v3.8h, v3.8h, v4.8h + sqrdmulh v20.8h, v2.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v3.8h, v16.8h + sqrdmulh v22.8h, v2.8h, v17.8h + sqrdmulh v23.8h, v3.8h, v17.8h + sqrdmulh v24.8h, v2.8h, v18.8h + sqrdmulh v25.8h, v3.8h, v18.8h + sqrdmulh v26.8h, v2.8h, v19.8h + sqrdmulh v27.8h, v3.8h, v19.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + add v22.8h, v22.8h, v4.8h + add v23.8h, v23.8h, v4.8h + add v24.8h, v24.8h, v4.8h + add v25.8h, v25.8h, v4.8h + add v26.8h, v26.8h, v4.8h + add v27.8h, v27.8h, v4.8h + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + st1 {v24.8h, v25.8h}, [x5], #32 + st1 {v26.8h, v27.8h}, [x8], #32 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x2, x2, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x8, x8, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_v_tbl): + .hword L(ipred_smooth_v_tbl) - 640b + .hword L(ipred_smooth_v_tbl) - 320b + .hword L(ipred_smooth_v_tbl) - 160b + .hword L(ipred_smooth_v_tbl) - 80b + .hword L(ipred_smooth_v_tbl) - 40b +endfunc + +// void ipred_smooth_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_16bpc_neon, export=1 + movrel x8, X(sm_weights) + add x8, x8, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_h_tbl) + add x12, x2, w3, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v5.8h}, [x12] // right + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1r {v7.2s}, [x8] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + ushll v7.8h, v7.8b, #7 // weights_hor << 7 +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 // left + zip1 v1.2d, v1.2d, v0.2d // left, flipped + zip1 v0.2d, v3.2d, v2.2d + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sqrdmulh v20.8h, v0.8h, v7.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v1.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + st1 {v20.d}[0], [x0], x1 + st1 {v20.d}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.d}[0], [x0], x1 + st1 {v21.d}[1], [x6], x1 + b.gt 4b + ret +80: + ld1 {v7.8b}, [x8] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + ushll v7.8h, v7.8b, #7 // weights_hor << 7 +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + sub v3.8h, v3.8h, v5.8h // left-right + sub v2.8h, v2.8h, v5.8h + sub v1.8h, v1.8h, v5.8h + sub v0.8h, v0.8h, v5.8h + sqrdmulh v20.8h, v3.8h, v7.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v2.8h, v7.8h // (left flipped) + sqrdmulh v22.8h, v1.8h, v7.8h + sqrdmulh v23.8h, v0.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + add v22.8h, v22.8h, v5.8h + add v23.8h, v23.8h, v5.8h + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + sub x2, x2, #8 + mov x7, #-8 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sub v2.8h, v2.8h, v5.8h + sub v3.8h, v3.8h, v5.8h +2: + ld1 {v7.16b}, [x8], #16 // weights_hor + ushll v6.8h, v7.8b, #7 // weights_hor << 7 + ushll2 v7.8h, v7.16b, #7 + sqrdmulh v20.8h, v3.8h, v6.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v3.8h, v7.8h // (left flipped) + sqrdmulh v22.8h, v2.8h, v6.8h + sqrdmulh v23.8h, v2.8h, v7.8h + sqrdmulh v24.8h, v1.8h, v6.8h + sqrdmulh v25.8h, v1.8h, v7.8h + sqrdmulh v26.8h, v0.8h, v6.8h + sqrdmulh v27.8h, v0.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + add v22.8h, v22.8h, v5.8h + add v23.8h, v23.8h, v5.8h + add v24.8h, v24.8h, v5.8h + add v25.8h, v25.8h, v5.8h + add v26.8h, v26.8h, v5.8h + add v27.8h, v27.8h, v5.8h + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + st1 {v24.8h, v25.8h}, [x5], #32 + st1 {v26.8h, v27.8h}, [x10], #32 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x8, x8, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_h_tbl): + .hword L(ipred_smooth_h_tbl) - 640b + .hword L(ipred_smooth_h_tbl) - 320b + .hword L(ipred_smooth_h_tbl) - 160b + .hword L(ipred_smooth_h_tbl) - 80b + .hword L(ipred_smooth_h_tbl) - 40b +endfunc + +// void ipred_filter_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height, +// const int bitdepth_max); +.macro filter_fn bpc +function ipred_filter_\bpc\()bpc_neon + and w5, w5, #511 + movrel x6, X(filter_intra_taps) + lsl w5, w5, #6 + add x6, x6, w5, uxtw + ld1 {v16.8b, v17.8b, v18.8b, v19.8b}, [x6], #32 + clz w9, w3 + adr x5, L(ipred_filter\bpc\()_tbl) + ld1 {v20.8b, v21.8b, v22.8b}, [x6] + sub w9, w9, #26 + ldrh w9, [x5, w9, uxtw #1] + sxtl v16.8h, v16.8b + sxtl v17.8h, v17.8b + sub x5, x5, w9, uxtw + sxtl v18.8h, v18.8b + sxtl v19.8h, v19.8b + add x6, x0, x1 + lsl x1, x1, #1 + sxtl v20.8h, v20.8b + sxtl v21.8h, v21.8b + sxtl v22.8h, v22.8b + dup v31.8h, w8 + movi v30.8h, #0 + br x5 +40: + ldur d0, [x2, #2] // top (0-3) + sub x2, x2, #4 + mov x7, #-4 +4: + ld1 {v1.4h}, [x2], x7 // left (0-1) + topleft (2) +.if \bpc == 10 + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + srshr v2.8h, v2.8h, #4 + smax v2.8h, v2.8h, v30.8h +.else + smull v2.4s, v17.4h, v0.h[0] // p1(top[0]) * filter(1) + smlal v2.4s, v18.4h, v0.h[1] // p2(top[1]) * filter(2) + smlal v2.4s, v19.4h, v0.h[2] // p3(top[2]) * filter(3) + smlal v2.4s, v20.4h, v0.h[3] // p4(top[3]) * filter(4) + smlal v2.4s, v16.4h, v1.h[2] // p0(topleft) * filter(0) + smlal v2.4s, v21.4h, v1.h[1] // p5(left[0]) * filter(5) + smlal v2.4s, v22.4h, v1.h[0] // p6(left[1]) * filter(6) + smull2 v3.4s, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + smlal2 v3.4s, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + smlal2 v3.4s, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + smlal2 v3.4s, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + smlal2 v3.4s, v16.8h, v1.h[2] // p0(topleft) * filter(0) + smlal2 v3.4s, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + smlal2 v3.4s, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + sqrshrun v2.4h, v2.4s, #4 + sqrshrun2 v2.8h, v3.4s, #4 +.endif + smin v2.8h, v2.8h, v31.8h + subs w4, w4, #2 + st1 {v2.d}[0], [x0], x1 + uxtl v0.8h, v2.8b + ext v0.16b, v2.16b, v2.16b, #8 // move top from [4-7] to [0-3] + st1 {v2.d}[1], [x6], x1 + b.gt 4b + ret +80: + ldur q0, [x2, #2] // top (0-7) + sub x2, x2, #4 + mov x7, #-4 +8: + ld1 {v1.4h}, [x2], x7 // left (0-1) + topleft (2) +.if \bpc == 10 + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + mul v3.8h, v17.8h, v0.h[4] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v0.h[5] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v0.h[6] // p3(top[2]) * filter(3) + srshr v2.8h, v2.8h, #4 + smax v2.8h, v2.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + mla v3.8h, v20.8h, v0.h[7] // p4(top[3]) * filter(4) + mla v3.8h, v16.8h, v0.h[3] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v2.h[3] // p5(left[0]) * filter(5) + mla v3.8h, v22.8h, v2.h[7] // p6(left[1]) * filter(6) + srshr v3.8h, v3.8h, #4 + smax v3.8h, v3.8h, v30.8h +.else + smull v2.4s, v17.4h, v0.h[0] // p1(top[0]) * filter(1) + smlal v2.4s, v18.4h, v0.h[1] // p2(top[1]) * filter(2) + smlal v2.4s, v19.4h, v0.h[2] // p3(top[2]) * filter(3) + smlal v2.4s, v20.4h, v0.h[3] // p4(top[3]) * filter(4) + smlal v2.4s, v16.4h, v1.h[2] // p0(topleft) * filter(0) + smlal v2.4s, v21.4h, v1.h[1] // p5(left[0]) * filter(5) + smlal v2.4s, v22.4h, v1.h[0] // p6(left[1]) * filter(6) + smull2 v3.4s, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + smlal2 v3.4s, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + smlal2 v3.4s, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + smlal2 v3.4s, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + smlal2 v3.4s, v16.8h, v1.h[2] // p0(topleft) * filter(0) + smlal2 v3.4s, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + smlal2 v3.4s, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + smull v4.4s, v17.4h, v0.h[4] // p1(top[0]) * filter(1) + smlal v4.4s, v18.4h, v0.h[5] // p2(top[1]) * filter(2) + smlal v4.4s, v19.4h, v0.h[6] // p3(top[2]) * filter(3) + sqrshrun v2.4h, v2.4s, #4 + sqrshrun2 v2.8h, v3.4s, #4 + smin v2.8h, v2.8h, v31.8h + smlal v4.4s, v20.4h, v0.h[7] // p4(top[3]) * filter(4) + smlal v4.4s, v16.4h, v0.h[3] // p0(topleft) * filter(0) + smlal v4.4s, v21.4h, v2.h[3] // p5(left[0]) * filter(5) + smlal v4.4s, v22.4h, v2.h[7] // p6(left[1]) * filter(6) + smull2 v5.4s, v17.8h, v0.h[4] // p1(top[0]) * filter(1) + smlal2 v5.4s, v18.8h, v0.h[5] // p2(top[1]) * filter(2) + smlal2 v5.4s, v19.8h, v0.h[6] // p3(top[2]) * filter(3) + smlal2 v5.4s, v20.8h, v0.h[7] // p4(top[3]) * filter(4) + smlal2 v5.4s, v16.8h, v0.h[3] // p0(topleft) * filter(0) + smlal2 v5.4s, v21.8h, v2.h[3] // p5(left[0]) * filter(5) + smlal2 v5.4s, v22.8h, v2.h[7] // p6(left[1]) * filter(6) + sqrshrun v3.4h, v4.4s, #4 + sqrshrun2 v3.8h, v5.4s, #4 +.endif + smin v3.8h, v3.8h, v31.8h + subs w4, w4, #2 + st2 {v2.d, v3.d}[0], [x0], x1 + zip2 v0.2d, v2.2d, v3.2d + st2 {v2.d, v3.d}[1], [x6], x1 + b.gt 8b + ret +160: +320: + add x8, x2, #2 + sub x2, x2, #4 + mov x7, #-4 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld1 {v0.4h}, [x2], x7 // left (0-1) + topleft (2) +2: + ld1 {v1.8h, v2.8h}, [x8], #32 // top(0-15) +.if \bpc == 10 + mul v3.8h, v16.8h, v0.h[2] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v0.h[1] // p5(left[0]) * filter(5) + mla v3.8h, v22.8h, v0.h[0] // p6(left[1]) * filter(6) + mla v3.8h, v17.8h, v1.h[0] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v1.h[1] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v1.h[2] // p3(top[2]) * filter(3) + mla v3.8h, v20.8h, v1.h[3] // p4(top[3]) * filter(4) + + mul v4.8h, v17.8h, v1.h[4] // p1(top[0]) * filter(1) + mla v4.8h, v18.8h, v1.h[5] // p2(top[1]) * filter(2) + mla v4.8h, v19.8h, v1.h[6] // p3(top[2]) * filter(3) + srshr v3.8h, v3.8h, #4 + smax v3.8h, v3.8h, v30.8h + smin v3.8h, v3.8h, v31.8h + mla v4.8h, v20.8h, v1.h[7] // p4(top[3]) * filter(4) + mla v4.8h, v16.8h, v1.h[3] // p0(topleft) * filter(0) + mla v4.8h, v21.8h, v3.h[3] // p5(left[0]) * filter(5) + mla v4.8h, v22.8h, v3.h[7] // p6(left[1]) * filter(6) + + mul v5.8h, v17.8h, v2.h[0] // p1(top[0]) * filter(1) + mla v5.8h, v18.8h, v2.h[1] // p2(top[1]) * filter(2) + mla v5.8h, v19.8h, v2.h[2] // p3(top[2]) * filter(3) + srshr v4.8h, v4.8h, #4 + smax v4.8h, v4.8h, v30.8h + smin v4.8h, v4.8h, v31.8h + mla v5.8h, v20.8h, v2.h[3] // p4(top[3]) * filter(4) + mla v5.8h, v16.8h, v1.h[7] // p0(topleft) * filter(0) + mla v5.8h, v21.8h, v4.h[3] // p5(left[0]) * filter(5) + mla v5.8h, v22.8h, v4.h[7] // p6(left[1]) * filter(6) + + mul v6.8h, v17.8h, v2.h[4] // p1(top[0]) * filter(1) + mla v6.8h, v18.8h, v2.h[5] // p2(top[1]) * filter(2) + mla v6.8h, v19.8h, v2.h[6] // p3(top[2]) * filter(3) + srshr v5.8h, v5.8h, #4 + smax v5.8h, v5.8h, v30.8h + smin v5.8h, v5.8h, v31.8h + mla v6.8h, v20.8h, v2.h[7] // p4(top[3]) * filter(4) + mla v6.8h, v16.8h, v2.h[3] // p0(topleft) * filter(0) + mla v6.8h, v21.8h, v5.h[3] // p5(left[0]) * filter(5) + mla v6.8h, v22.8h, v5.h[7] // p6(left[1]) * filter(6) + + subs w3, w3, #16 + srshr v6.8h, v6.8h, #4 + smax v6.8h, v6.8h, v30.8h +.else + smull v3.4s, v16.4h, v0.h[2] // p0(topleft) * filter(0) + smlal v3.4s, v21.4h, v0.h[1] // p5(left[0]) * filter(5) + smlal v3.4s, v22.4h, v0.h[0] // p6(left[1]) * filter(6) + smlal v3.4s, v17.4h, v1.h[0] // p1(top[0]) * filter(1) + smlal v3.4s, v18.4h, v1.h[1] // p2(top[1]) * filter(2) + smlal v3.4s, v19.4h, v1.h[2] // p3(top[2]) * filter(3) + smlal v3.4s, v20.4h, v1.h[3] // p4(top[3]) * filter(4) + smull2 v4.4s, v16.8h, v0.h[2] // p0(topleft) * filter(0) + smlal2 v4.4s, v21.8h, v0.h[1] // p5(left[0]) * filter(5) + smlal2 v4.4s, v22.8h, v0.h[0] // p6(left[1]) * filter(6) + smlal2 v4.4s, v17.8h, v1.h[0] // p1(top[0]) * filter(1) + smlal2 v4.4s, v18.8h, v1.h[1] // p2(top[1]) * filter(2) + smlal2 v4.4s, v19.8h, v1.h[2] // p3(top[2]) * filter(3) + smlal2 v4.4s, v20.8h, v1.h[3] // p4(top[3]) * filter(4) + + smull v5.4s, v17.4h, v1.h[4] // p1(top[0]) * filter(1) + smlal v5.4s, v18.4h, v1.h[5] // p2(top[1]) * filter(2) + smlal v5.4s, v19.4h, v1.h[6] // p3(top[2]) * filter(3) + sqrshrun v3.4h, v3.4s, #4 + sqrshrun2 v3.8h, v4.4s, #4 + smin v3.8h, v3.8h, v31.8h + smlal v5.4s, v20.4h, v1.h[7] // p4(top[3]) * filter(4) + smlal v5.4s, v16.4h, v1.h[3] // p0(topleft) * filter(0) + smlal v5.4s, v21.4h, v3.h[3] // p5(left[0]) * filter(5) + smlal v5.4s, v22.4h, v3.h[7] // p6(left[1]) * filter(6) + smull2 v6.4s, v17.8h, v1.h[4] // p1(top[0]) * filter(1) + smlal2 v6.4s, v18.8h, v1.h[5] // p2(top[1]) * filter(2) + smlal2 v6.4s, v19.8h, v1.h[6] // p3(top[2]) * filter(3) + smlal2 v6.4s, v20.8h, v1.h[7] // p4(top[3]) * filter(4) + smlal2 v6.4s, v16.8h, v1.h[3] // p0(topleft) * filter(0) + smlal2 v6.4s, v21.8h, v3.h[3] // p5(left[0]) * filter(5) + smlal2 v6.4s, v22.8h, v3.h[7] // p6(left[1]) * filter(6) + + smull v24.4s, v17.4h, v2.h[0] // p1(top[0]) * filter(1) + smlal v24.4s, v18.4h, v2.h[1] // p2(top[1]) * filter(2) + smlal v24.4s, v19.4h, v2.h[2] // p3(top[2]) * filter(3) + sqrshrun v4.4h, v5.4s, #4 + sqrshrun2 v4.8h, v6.4s, #4 + smin v4.8h, v4.8h, v31.8h + smlal v24.4s, v20.4h, v2.h[3] // p4(top[3]) * filter(4) + smlal v24.4s, v16.4h, v1.h[7] // p0(topleft) * filter(0) + smlal v24.4s, v21.4h, v4.h[3] // p5(left[0]) * filter(5) + smlal v24.4s, v22.4h, v4.h[7] // p6(left[1]) * filter(6) + smull2 v25.4s, v17.8h, v2.h[0] // p1(top[0]) * filter(1) + smlal2 v25.4s, v18.8h, v2.h[1] // p2(top[1]) * filter(2) + smlal2 v25.4s, v19.8h, v2.h[2] // p3(top[2]) * filter(3) + smlal2 v25.4s, v20.8h, v2.h[3] // p4(top[3]) * filter(4) + smlal2 v25.4s, v16.8h, v1.h[7] // p0(topleft) * filter(0) + smlal2 v25.4s, v21.8h, v4.h[3] // p5(left[0]) * filter(5) + smlal2 v25.4s, v22.8h, v4.h[7] // p6(left[1]) * filter(6) + + smull v26.4s, v17.4h, v2.h[4] // p1(top[0]) * filter(1) + smlal v26.4s, v18.4h, v2.h[5] // p2(top[1]) * filter(2) + smlal v26.4s, v19.4h, v2.h[6] // p3(top[2]) * filter(3) + sqrshrun v5.4h, v24.4s, #4 + sqrshrun2 v5.8h, v25.4s, #4 + smin v5.8h, v5.8h, v31.8h + smlal v26.4s, v20.4h, v2.h[7] // p4(top[3]) * filter(4) + smlal v26.4s, v16.4h, v2.h[3] // p0(topleft) * filter(0) + smlal v26.4s, v21.4h, v5.h[3] // p5(left[0]) * filter(5) + smlal v26.4s, v22.4h, v5.h[7] // p6(left[1]) * filter(6) + smull2 v27.4s, v17.8h, v2.h[4] // p1(top[0]) * filter(1) + smlal2 v27.4s, v18.8h, v2.h[5] // p2(top[1]) * filter(2) + smlal2 v27.4s, v19.8h, v2.h[6] // p3(top[2]) * filter(3) + smlal2 v27.4s, v20.8h, v2.h[7] // p4(top[3]) * filter(4) + smlal2 v27.4s, v16.8h, v2.h[3] // p0(topleft) * filter(0) + smlal2 v27.4s, v21.8h, v5.h[3] // p5(left[0]) * filter(5) + smlal2 v27.4s, v22.8h, v5.h[7] // p6(left[1]) * filter(6) + + subs w3, w3, #16 + sqrshrun v6.4h, v26.4s, #4 + sqrshrun2 v6.8h, v27.4s, #4 +.endif + smin v6.8h, v6.8h, v31.8h + + ins v0.h[2], v2.h[7] + st4 {v3.d, v4.d, v5.d, v6.d}[0], [x0], #32 + ins v0.h[0], v6.h[7] + st4 {v3.d, v4.d, v5.d, v6.d}[1], [x6], #32 + ins v0.h[1], v6.h[3] + b.gt 2b + subs w4, w4, #2 + b.le 9f + sub x8, x6, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_filter\bpc\()_tbl): + .hword L(ipred_filter\bpc\()_tbl) - 320b + .hword L(ipred_filter\bpc\()_tbl) - 160b + .hword L(ipred_filter\bpc\()_tbl) - 80b + .hword L(ipred_filter\bpc\()_tbl) - 40b +endfunc +.endm + +filter_fn 10 +filter_fn 12 + +function ipred_filter_16bpc_neon, export=1 + ldr w8, [sp] + cmp w8, 0x3ff + b.le ipred_filter_10bpc_neon + b ipred_filter_12bpc_neon +endfunc + +// void pal_pred_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_16bpc_neon, export=1 + ld1 {v30.8h}, [x2] + clz w9, w4 + adr x6, L(pal_pred_tbl) + sub w9, w9, #25 + ldrh w9, [x6, w9, uxtw #1] + movi v31.8h, #1, lsl #8 + sub x6, x6, w9, uxtw + br x6 +40: + add x2, x0, x1 + lsl x1, x1, #1 +4: + ld1 {v1.16b}, [x3], #16 + subs w5, w5, #4 + // Restructure v1 from a, b, c, ... into 2*a, 2*a+1, 2*b, 2*b+1, 2*c, 2*c+1, ... + add v1.16b, v1.16b, v1.16b + zip1 v0.16b, v1.16b, v1.16b + zip2 v1.16b, v1.16b, v1.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + st1 {v0.d}[0], [x0], x1 + tbl v1.16b, {v30.16b}, v1.16b + st1 {v0.d}[1], [x2], x1 + st1 {v1.d}[0], [x0], x1 + st1 {v1.d}[1], [x2], x1 + b.gt 4b + ret +80: + add x2, x0, x1 + lsl x1, x1, #1 +8: + ld1 {v2.16b, v3.16b}, [x3], #32 + subs w5, w5, #4 + add v2.16b, v2.16b, v2.16b + add v3.16b, v3.16b, v3.16b + zip1 v0.16b, v2.16b, v2.16b + zip2 v1.16b, v2.16b, v2.16b + zip1 v2.16b, v3.16b, v3.16b + zip2 v3.16b, v3.16b, v3.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + tbl v1.16b, {v30.16b}, v1.16b + st1 {v0.8h}, [x0], x1 + tbl v2.16b, {v30.16b}, v2.16b + st1 {v1.8h}, [x2], x1 + tbl v3.16b, {v30.16b}, v3.16b + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x2], x1 + b.gt 8b + ret +160: + add x2, x0, x1 + lsl x1, x1, #1 +16: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #4 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + st1 {v2.8h, v3.8h}, [x2], x1 + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h}, [x0], x1 + st1 {v6.8h, v7.8h}, [x2], x1 + b.gt 16b + ret +320: + add x2, x0, x1 + lsl x1, x1, #1 +32: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #2 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x1 + b.gt 32b + ret +640: + add x2, x0, #64 +64: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #1 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x1 + b.gt 64b + ret + +L(pal_pred_tbl): + .hword L(pal_pred_tbl) - 640b + .hword L(pal_pred_tbl) - 320b + .hword L(pal_pred_tbl) - 160b + .hword L(pal_pred_tbl) - 80b + .hword L(pal_pred_tbl) - 40b +endfunc + +// void ipred_cfl_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_128_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + clz w9, w3 + adr x7, L(ipred_cfl_128_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + urshr v0.8h, v31.8h, #1 + dup v1.8h, w6 // alpha + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 +L(ipred_cfl_splat_w4): + ld1 {v4.8h, v5.8h}, [x5], #32 + subs w4, w4, #4 + smull v2.4s, v4.4h, v1.4h // diff = ac * alpha + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + sshr v16.4s, v2.4s, #31 // sign = diff >> 31 + sshr v17.4s, v3.4s, #31 + sshr v18.4s, v4.4s, #31 + sshr v19.4s, v5.4s, #31 + add v2.4s, v2.4s, v16.4s // diff + sign + add v3.4s, v3.4s, v17.4s + add v4.4s, v4.4s, v18.4s + add v5.4s, v5.4s, v19.4s + rshrn v2.4h, v2.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v2.d}[0], [x0], x1 + st1 {v2.d}[1], [x6], x1 + st1 {v3.d}[0], [x0], x1 + st1 {v3.d}[1], [x6], x1 + b.gt L(ipred_cfl_splat_w4) + ret +L(ipred_cfl_splat_w8): + ld1 {v4.8h, v5.8h}, [x5], #32 + subs w4, w4, #2 + smull v2.4s, v4.4h, v1.4h // diff = ac * alpha + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + sshr v16.4s, v2.4s, #31 // sign = diff >> 31 + sshr v17.4s, v3.4s, #31 + sshr v18.4s, v4.4s, #31 + sshr v19.4s, v5.4s, #31 + add v2.4s, v2.4s, v16.4s // diff + sign + add v3.4s, v3.4s, v17.4s + add v4.4s, v4.4s, v18.4s + add v5.4s, v5.4s, v19.4s + rshrn v2.4h, v2.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x6], x1 + b.gt L(ipred_cfl_splat_w8) + ret +L(ipred_cfl_splat_w16): + add x7, x5, w3, uxtw #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 +1: + ld1 {v2.8h, v3.8h}, [x5], #32 + ld1 {v4.8h, v5.8h}, [x7], #32 + subs w3, w3, #16 + smull v16.4s, v2.4h, v1.4h // diff = ac * alpha + smull2 v17.4s, v2.8h, v1.8h + smull v18.4s, v3.4h, v1.4h + smull2 v19.4s, v3.8h, v1.8h + smull v2.4s, v4.4h, v1.4h + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + sshr v20.4s, v16.4s, #31 // sign = diff >> 31 + sshr v21.4s, v17.4s, #31 + sshr v22.4s, v18.4s, #31 + sshr v23.4s, v19.4s, #31 + sshr v24.4s, v2.4s, #31 + sshr v25.4s, v3.4s, #31 + sshr v26.4s, v4.4s, #31 + sshr v27.4s, v5.4s, #31 + add v16.4s, v16.4s, v20.4s // diff + sign + add v17.4s, v17.4s, v21.4s + add v18.4s, v18.4s, v22.4s + add v19.4s, v19.4s, v23.4s + add v2.4s, v2.4s, v24.4s + add v3.4s, v3.4s, v25.4s + add v4.4s, v4.4s, v26.4s + add v5.4s, v5.4s, v27.4s + rshrn v16.4h, v16.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v16.8h, v17.4s, #6 + rshrn v17.4h, v18.4s, #6 + rshrn2 v17.8h, v19.4s, #6 + rshrn v6.4h, v2.4s, #6 + rshrn2 v6.8h, v3.4s, #6 + rshrn v7.4h, v4.4s, #6 + rshrn2 v7.8h, v5.4s, #6 + add v2.8h, v16.8h, v0.8h // dc + apply_sign() + add v3.8h, v17.8h, v0.8h + add v4.8h, v6.8h, v0.8h + add v5.8h, v7.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smax v4.8h, v4.8h, v30.8h + smax v5.8h, v5.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + smin v4.8h, v4.8h, v31.8h + smin v5.8h, v5.8h, v31.8h + st1 {v2.8h, v3.8h}, [x0], #32 + st1 {v4.8h, v5.8h}, [x6], #32 + b.gt 1b + subs w4, w4, #2 + add x5, x5, w9, uxtw #1 + add x7, x7, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b.gt 1b + ret + +L(ipred_cfl_128_tbl): +L(ipred_cfl_splat_tbl): + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w8) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w4) +endfunc + +// void ipred_cfl_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_top_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + clz w9, w3 + adr x7, L(ipred_cfl_top_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + dup v1.8h, w6 // alpha + add x2, x2, #2 + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 +4: + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) +8: + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) +16: + ld1 {v2.8h, v3.8h}, [x2] + addp v0.8h, v2.8h, v3.8h + addv h0, v0.8h + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) +32: + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + rshrn v0.4h, v0.4s, #5 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_top_tbl): + .hword L(ipred_cfl_top_tbl) - 32b + .hword L(ipred_cfl_top_tbl) - 16b + .hword L(ipred_cfl_top_tbl) - 8b + .hword L(ipred_cfl_top_tbl) - 4b +endfunc + +// void ipred_cfl_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_left_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + sub x2, x2, w4, uxtw #1 + clz w9, w3 + clz w8, w4 + adr x10, L(ipred_cfl_splat_tbl) + adr x7, L(ipred_cfl_left_tbl) + sub w9, w9, #26 + sub w8, w8, #26 + ldrh w9, [x10, w9, uxtw #1] + ldrh w8, [x7, w8, uxtw #1] + dup v1.8h, w6 // alpha + sub x9, x10, w9, uxtw + sub x7, x7, w8, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 + +L(ipred_cfl_left_h4): + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h8): + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h16): + ld1 {v2.8h, v3.8h}, [x2] + addp v0.8h, v2.8h, v3.8h + addv h0, v0.8h + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h32): + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + rshrn v0.4h, v0.4s, #5 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_tbl): + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h32) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h16) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h8) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h4) +endfunc + +// void ipred_cfl_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + sub x2, x2, w4, uxtw #1 + add w8, w3, w4 // width + height + dup v1.8h, w6 // alpha + clz w9, w3 + clz w6, w4 + dup v16.4s, w8 // width + height + adr x7, L(ipred_cfl_tbl) + rbit w8, w8 // rbit(width + height) + sub w9, w9, #22 // 22 leading bits, minus table offset 4 + sub w6, w6, #26 + clz w8, w8 // ctz(width + height) + ldrh w9, [x7, w9, uxtw #1] + ldrh w6, [x7, w6, uxtw #1] + neg w8, w8 // -ctz(width + height) + sub x9, x7, w9, uxtw + sub x7, x7, w6, uxtw + ushr v16.4s, v16.4s, #1 // (width + height) >> 1 + dup v17.4s, w8 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 + +L(ipred_cfl_h4): + ld1 {v0.4h}, [x2], #8 + uaddlv s0, v0.4h + br x9 +L(ipred_cfl_w4): + add x2, x2, #2 + ld1 {v2.4h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s2, v2.4h + cmp w4, #4 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) + +L(ipred_cfl_h8): + ld1 {v0.8h}, [x2], #16 + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w8): + add x2, x2, #2 + ld1 {v2.8h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s2, v2.8h + cmp w4, #8 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) + +L(ipred_cfl_h16): + ld1 {v2.8h, v3.8h}, [x2], #32 + addp v0.8h, v2.8h, v3.8h + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w16): + add x2, x2, #2 + ld1 {v2.8h, v3.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v2.8h, v2.8h, v3.8h + uaddlv s2, v2.8h + cmp w4, #16 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/8/32 + tst w4, #(32+16+8) // 16 added to make a consecutive bitmask + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_h32): + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2], #64 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w32): + add x2, x2, #2 + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + add v0.4s, v0.4s, v16.4s + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v2.8h, v2.8h, v4.8h + cmp w4, #32 + uaddlv s2, v2.8h + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #8 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_tbl): + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h4) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w4) +endfunc + +// void cfl_ac_420_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_420_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_420_w4): +1: // Copy and subsample input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v2.8h + addp v1.8h, v1.8h, v3.8h + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + subs w8, w8, #2 + st1 {v0.8h}, [x0], #16 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + b.gt 1b + trn2 v1.2d, v0.2d, v0.2d + trn2 v0.2d, v0.2d, v0.2d +L(ipred_cfl_ac_420_w4_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 2b +3: +L(ipred_cfl_ac_420_w4_calc_subtract_dc): + // Aggregate the sums + add v24.4s, v24.4s, v25.4s + add v26.4s, v26.4s, v27.4s + add v0.4s, v24.4s, v26.4s + addv s0, v0.4s // sum + sub x0, x0, w6, uxtw #3 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz + dup v4.8h, v4.h[0] +6: // Subtract dc from ac + ld1 {v0.8h, v1.8h}, [x0] + subs w6, w6, #4 + sub v0.8h, v0.8h, v4.8h + sub v1.8h, v1.8h, v4.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 6b + ret + +L(ipred_cfl_ac_420_w8): + cbnz w3, L(ipred_cfl_ac_420_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + add v0.8h, v0.8h, v2.8h + add v4.8h, v4.8h, v6.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v4.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + mov v0.16b, v1.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_420_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v2.8h + addp v1.8h, v1.8h, v3.8h + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + dup v1.4h, v0.h[3] + dup v3.4h, v0.h[7] + trn2 v2.2d, v0.2d, v0.2d + subs w8, w8, #2 + st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw v25.4s, v25.4s, v1.4h + uaddw v26.4s, v26.4s, v2.4h + uaddw v27.4s, v27.4s, v3.4h + b.gt 1b + trn1 v0.2d, v2.2d, v3.2d + trn1 v1.2d, v2.2d, v3.2d + +L(ipred_cfl_ac_420_w8_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 2b +3: + + // Double the height and reuse the w4 summing/subtracting + lsl w6, w6, #1 + lsl w9, w9, #1 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_w16): + adr x7, L(ipred_cfl_ac_420_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_420_w16_wpad0): +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x1], x2 + add v0.8h, v0.8h, v4.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x10], x2 + add v2.8h, v2.8h, v6.8h + addp v16.8h, v16.8h, v17.8h + addp v18.8h, v18.8h, v19.8h + addp v20.8h, v20.8h, v21.8h + addp v22.8h, v22.8h, v23.8h + add v16.8h, v16.8h, v20.8h + add v18.8h, v18.8h, v22.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v2.8h, #1 + shl v2.8h, v16.8h, #1 + shl v3.8h, v18.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad1): +1: // Copy and subsample input, padding 4 + ldr q2, [x1, #32] + ld1 {v0.8h, v1.8h}, [x1], x2 + ldr q5, [x10, #32] + ld1 {v3.8h, v4.8h}, [x10], x2 + addp v2.8h, v2.8h, v2.8h + addp v0.8h, v0.8h, v1.8h + addp v5.8h, v5.8h, v5.8h + addp v3.8h, v3.8h, v4.8h + ldr q18, [x1, #32] + add v2.4h, v2.4h, v5.4h + ld1 {v16.8h, v17.8h}, [x1], x2 + add v0.8h, v0.8h, v3.8h + ldr q21, [x10, #32] + ld1 {v19.8h, v20.8h}, [x10], x2 + addp v18.8h, v18.8h, v18.8h + addp v16.8h, v16.8h, v17.8h + addp v21.8h, v21.8h, v21.8h + addp v19.8h, v19.8h, v20.8h + add v18.4h, v18.4h, v21.4h + add v16.8h, v16.8h, v19.8h + shl v1.4h, v2.4h, #1 + shl v0.8h, v0.8h, #1 + shl v3.4h, v18.4h, #1 + shl v2.8h, v16.8h, #1 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad2): +1: // Copy and subsample input, padding 8 + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + add v0.8h, v0.8h, v2.8h + add v4.8h, v4.8h, v6.8h + shl v0.8h, v0.8h, #1 + shl v2.8h, v4.8h, #1 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad3): +1: // Copy and subsample input, padding 12 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + ld1 {v4.8h}, [x1], x2 + ld1 {v6.8h}, [x10], x2 + addp v0.8h, v0.8h, v4.8h + addp v2.8h, v2.8h, v6.8h + add v0.8h, v0.8h, v2.8h + shl v0.8h, v0.8h, #1 + dup v1.8h, v0.h[3] + dup v3.8h, v0.h[7] + trn2 v2.2d, v0.2d, v3.2d + trn1 v0.2d, v0.2d, v1.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 2b +3: + + // Quadruple the height and reuse the w4 summing/subtracting + lsl w6, w6, #2 + lsl w9, w9, #2 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_tbl): + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w16) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w8) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w4) + .hword 0 + +L(ipred_cfl_ac_420_w16_tbl): + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad0) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad1) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad2) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad3) +endfunc + +// void cfl_ac_422_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_422_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_422_w4): +1: // Copy and subsample input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + subs w8, w8, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_422_w8): + cbnz w3, L(ipred_cfl_ac_422_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + shl v2.8h, v4.8h, #2 + shl v3.8h, v6.8h, #2 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v4.4h, v0.h[3] + dup v5.8h, v0.h[7] + dup v6.4h, v2.h[3] + dup v7.8h, v2.h[7] + trn2 v1.2d, v0.2d, v5.2d + trn1 v0.2d, v0.2d, v4.2d + trn2 v3.2d, v2.2d, v7.2d + trn1 v2.2d, v2.2d, v6.2d + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w16): + adr x7, L(ipred_cfl_ac_422_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_422_w16_wpad0): +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + shl v2.8h, v4.8h, #2 + shl v3.8h, v6.8h, #2 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad1): +1: // Copy and subsample input, padding 4 + ldr q2, [x1, #32] + ld1 {v0.8h, v1.8h}, [x1], x2 + ldr q6, [x10, #32] + ld1 {v4.8h, v5.8h}, [x10], x2 + addp v2.8h, v2.8h, v2.8h + addp v0.8h, v0.8h, v1.8h + addp v6.8h, v6.8h, v6.8h + addp v4.8h, v4.8h, v5.8h + shl v1.4h, v2.4h, #2 + shl v0.8h, v0.8h, #2 + shl v3.4h, v6.4h, #2 + shl v2.8h, v4.8h, #2 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad2): +1: // Copy and subsample input, padding 8 + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad3): +1: // Copy and subsample input, padding 12 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + addp v0.8h, v0.8h, v0.8h + addp v2.8h, v2.8h, v2.8h + shl v0.4h, v0.4h, #2 + shl v2.4h, v2.4h, #2 + dup v1.8h, v0.h[3] + dup v3.8h, v2.h[3] + trn1 v0.2d, v0.2d, v1.2d + trn1 v2.2d, v2.2d, v3.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_tbl): + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w16) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w8) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w4) + .hword 0 + +L(ipred_cfl_ac_422_w16_tbl): + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad0) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad1) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad2) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad3) +endfunc diff --git a/third_party/dav1d/src/arm/64/itx.S b/third_party/dav1d/src/arm/64/itx.S --- a/third_party/dav1d/src/arm/64/itx.S +++ b/third_party/dav1d/src/arm/64/itx.S @@ -159,35 +159,10 @@ sqrdmulh \r6\sz, \r6\sz, \c sqrdmulh \r7\sz, \r7\sz, \c .endif .endm -.macro scale_wide sz, c, r0, r1, r2 r3, r4, r5, r6, r7 - smull_sz v2, v3, \r0, \c, \sz - smull_sz v4, v5, \r1, \c, \sz - smull_sz v6, v7, \r2, \c, \sz - rshrn_sz \r0, v2, v3, #12, \sz - smull_sz v2, v3, \r3, \c, \sz - rshrn_sz \r1, v4, v5, #12, \sz -.ifnb \r4 - smull_sz v4, v5, \r4, \c, \sz -.endif - rshrn_sz \r2, v6, v7, #12, \sz -.ifnb \r4 - smull_sz v6, v7, \r5, \c, \sz -.endif - rshrn_sz \r3, v2, v3, #12, \sz -.ifnb \r4 - smull_sz v2, v3, \r6, \c, \sz - rshrn_sz \r4, v4, v5, #12, \sz - smull_sz v4, v5, \r7, \c, \sz - rshrn_sz \r5, v6, v7, #12, \sz - rshrn_sz \r6, v2, v3, #12, \sz - rshrn_sz \r7, v4, v5, #12, \sz -.endif -.endm - .macro load_add_store load, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src, shiftbits=4 .ifnb \load ld1 {\load}, [\src], x1 .endif .ifnb \shift @@ -597,45 +572,44 @@ iadst_8x4 v19, v18, v17, v16 ret endfunc function inv_identity_4x4_neon - mov w16, #5793 + mov w16, #(5793-4096)*8 dup v0.4h, w16 - smull v4.4s, v16.4h, v0.h[0] - smull v5.4s, v17.4h, v0.h[0] - smull v6.4s, v18.4h, v0.h[0] - smull v7.4s, v19.4h, v0.h[0] - rshrn v16.4h, v4.4s, #12 - rshrn v17.4h, v5.4s, #12 - rshrn v18.4h, v6.4s, #12 - rshrn v19.4h, v7.4s, #12 + sqrdmulh v4.4h, v16.4h, v0.h[0] + sqrdmulh v5.4h, v17.4h, v0.h[0] + sqrdmulh v6.4h, v18.4h, v0.h[0] + sqrdmulh v7.4h, v19.4h, v0.h[0] + sqadd v16.4h, v16.4h, v4.4h + sqadd v17.4h, v17.4h, v5.4h + sqadd v18.4h, v18.4h, v6.4h + sqadd v19.4h, v19.4h, v7.4h ret endfunc function inv_identity_8x4_neon - mov w16, #5793 + mov w16, #(5793-4096)*8 dup v0.4h, w16 - smull v2.4s, v16.4h, v0.h[0] - smull2 v3.4s, v16.8h, v0.h[0] - smull v4.4s, v17.4h, v0.h[0] - smull2 v5.4s, v17.8h, v0.h[0] - rshrn v16.4h, v2.4s, #12 - rshrn2 v16.8h, v3.4s, #12 - smull v6.4s, v18.4h, v0.h[0] - smull2 v7.4s, v18.8h, v0.h[0] - rshrn v17.4h, v4.4s, #12 - rshrn2 v17.8h, v5.4s, #12 - smull v2.4s, v19.4h, v0.h[0] - smull2 v3.4s, v19.8h, v0.h[0] - rshrn v18.4h, v6.4s, #12 - rshrn2 v18.8h, v7.4s, #12 - rshrn v19.4h, v2.4s, #12 - rshrn2 v19.8h, v3.4s, #12 + sqrdmulh v4.8h, v16.8h, v0.h[0] + sqrdmulh v5.8h, v17.8h, v0.h[0] + sqrdmulh v6.8h, v18.8h, v0.h[0] + sqrdmulh v7.8h, v19.8h, v0.h[0] + sqadd v16.8h, v16.8h, v4.8h + sqadd v17.8h, v17.8h, v5.8h + sqadd v18.8h, v18.8h, v6.8h + sqadd v19.8h, v19.8h, v7.8h ret endfunc +.macro identity_8x4_shift1 r0, r1, r2, r3, c +.irp i, \r0\().8h, \r1\().8h, \r2\().8h, \r3\().8h + sqrdmulh v2.8h, \i, \c + srhadd \i, \i, v2.8h +.endr +.endm + function inv_txfm_add_wht_wht_4x4_neon, export=1 mov x15, x30 movi v31.8h, #0 ld1 {v16.4h,v17.4h,v18.4h,v19.4h}, [x2] st1 {v31.8h}, [x2], #16 @@ -875,72 +849,85 @@ iadst_8 v23, v22, v21, v20, v19, v18, v17, v16, .4h ret endfunc function inv_identity_8x8_neon - shl v16.8h, v16.8h, #1 - shl v17.8h, v17.8h, #1 - shl v18.8h, v18.8h, #1 - shl v19.8h, v19.8h, #1 - shl v20.8h, v20.8h, #1 - shl v21.8h, v21.8h, #1 - shl v22.8h, v22.8h, #1 - shl v23.8h, v23.8h, #1 + sqshl v16.8h, v16.8h, #1 + sqshl v17.8h, v17.8h, #1 + sqshl v18.8h, v18.8h, #1 + sqshl v19.8h, v19.8h, #1 + sqshl v20.8h, v20.8h, #1 + sqshl v21.8h, v21.8h, #1 + sqshl v22.8h, v22.8h, #1 + sqshl v23.8h, v23.8h, #1 ret endfunc function inv_identity_4x8_neon - shl v16.4h, v16.4h, #1 - shl v17.4h, v17.4h, #1 - shl v18.4h, v18.4h, #1 - shl v19.4h, v19.4h, #1 - shl v20.4h, v20.4h, #1 - shl v21.4h, v21.4h, #1 - shl v22.4h, v22.4h, #1 - shl v23.4h, v23.4h, #1 + sqshl v16.4h, v16.4h, #1 + sqshl v17.4h, v17.4h, #1 + sqshl v18.4h, v18.4h, #1 + sqshl v19.4h, v19.4h, #1 + sqshl v20.4h, v20.4h, #1 + sqshl v21.4h, v21.4h, #1 + sqshl v22.4h, v22.4h, #1 + sqshl v23.4h, v23.4h, #1 ret endfunc -function inv_txfm_add_8x8_neon +.macro def_fn_8x8_base variant +function inv_txfm_\variant\()add_8x8_neon movi v28.8h, #0 movi v29.8h, #0 movi v30.8h, #0 movi v31.8h, #0 ld1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x2] st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x2], #64 ld1 {v20.8h,v21.8h,v22.8h,v23.8h}, [x2] st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x2] +.ifc \variant, identity_ + // The identity shl #1 and downshift srshr #1 cancel out +.else blr x4 srshr v16.8h, v16.8h, #1 srshr v17.8h, v17.8h, #1 srshr v18.8h, v18.8h, #1 srshr v19.8h, v19.8h, #1 srshr v20.8h, v20.8h, #1 srshr v21.8h, v21.8h, #1 srshr v22.8h, v22.8h, #1 srshr v23.8h, v23.8h, #1 +.endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v24, v25 blr x5 load_add_store_8x8 x0, x7 br x15 endfunc +.endm + +def_fn_8x8_base +def_fn_8x8_base identity_ .macro def_fn_8x8 txfm1, txfm2 function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_neon, export=1 mov x15, x30 .ifc \txfm1\()_\txfm2, dct_dct idct_dc 8, 8, 1 .endif + adr x5, inv_\txfm2\()_8x8_neon +.ifc \txfm1, identity + b inv_txfm_identity_add_8x8_neon +.else adr x4, inv_\txfm1\()_8x8_neon - adr x5, inv_\txfm2\()_8x8_neon b inv_txfm_add_8x8_neon +.endif endfunc .endm def_fn_8x8 dct, dct def_fn_8x8 identity, identity @@ -1081,13 +1068,16 @@ smull_smlal v6, v7, v3, v2, v0.h[3], v0.h[2], \sz // -> t14a rshrn_sz v21, v4, v5, #12, \sz // t9a rshrn_sz v27, v6, v7, #12, \sz // t14a smull_smlsl v4, v5, v29, v23, v0.h[2], v0.h[3], \sz // -> t13a - neg v29\sz, v29\sz - smull_smlsl v6, v7, v29, v23, v0.h[3], v0.h[2], \sz // -> t10a + smull_smlal v6, v7, v29, v23, v0.h[3], v0.h[2], \sz // -> t10a rshrn_sz v29, v4, v5, #12, \sz // t13a + neg v6.4s, v6.4s +.ifc \sz, .8h + neg v7.4s, v7.4s +.endif rshrn_sz v23, v6, v7, #12, \sz // t10a sqsub v2\sz, v17\sz, v19\sz // t11a sqadd v17\sz, v17\sz, v19\sz // t8a sqsub v3\sz, v31\sz, v25\sz // t12a @@ -1331,31 +1321,63 @@ iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, .4h, .8b ret endfunc function inv_identity_8x16_neon - mov w16, #2*5793 + mov w16, #2*(5793-4096)*8 dup v0.4h, w16 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - smull v2.4s, v\i\().4h, v0.h[0] - smull2 v3.4s, v\i\().8h, v0.h[0] - rshrn v\i\().4h, v2.4s, #12 - rshrn2 v\i\().8h, v3.4s, #12 + sqrdmulh v2.8h, v\i\().8h, v0.h[0] + sqadd v\i\().8h, v\i\().8h, v\i\().8h + sqadd v\i\().8h, v\i\().8h, v2.8h .endr ret endfunc function inv_identity_4x16_neon - mov w16, #2*5793 + mov w16, #2*(5793-4096)*8 dup v0.4h, w16 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - smull v2.4s, v\i\().4h, v0.h[0] - rshrn v\i\().4h, v2.4s, #12 + sqrdmulh v2.4h, v\i\().4h, v0.h[0] + sqadd v\i\().4h, v\i\().4h, v\i\().4h + sqadd v\i\().4h, v\i\().4h, v2.4h .endr ret endfunc +.macro identity_8x16_shift2 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + sqrdmulh v2.8h, \i, \c + sshr v2.8h, v2.8h, #1 + srhadd \i, \i, v2.8h +.endr +.endm + +.macro identity_8x16_shift1 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + sqrdmulh v2.8h, \i, \c + srshr v2.8h, v2.8h, #1 + sqadd \i, \i, v2.8h +.endr +.endm + +.macro identity_8x8_shift1 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + sqrdmulh v2.8h, \i, \c + srshr v2.8h, v2.8h, #1 + sqadd \i, \i, v2.8h +.endr +.endm + +.macro identity_8x8 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + sqrdmulh v2.8h, \i, \c + sqadd \i, \i, \i + sqadd \i, \i, v2.8h +.endr +.endm + function inv_txfm_horz_16x8_neon mov x14, x30 movi v7.8h, #0 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ld1 {v\i\().8h}, [x7] @@ -1373,10 +1395,30 @@ .endr br x14 endfunc +function inv_txfm_horz_identity_16x8_neon + mov x14, x30 + movi v7.8h, #0 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7] + st1 {v7.8h}, [x7], x8 +.endr + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 + identity_8x16_shift2 v0.h[0] + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v4, v5 + +.irp i, 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 + st1 {v\i\().8h}, [x6], #16 +.endr + + br x14 +endfunc + function inv_txfm_horz_scale_16x8_neon mov x14, x30 movi v7.8h, #0 mov w16, #2896*8 dup v0.4h, w16 @@ -1419,11 +1461,11 @@ cmp w3, w13 b.lt 1f .endif add x7, x2, #(\i*2) mov x8, #16*2 - bl inv_txfm_horz_16x8_neon + blr x9 .endr b 2f 1: movi v4.8h, #0 movi v5.8h, #0 @@ -1447,11 +1489,16 @@ .macro def_fn_16x16 txfm1, txfm2, eob_half function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_neon, export=1 .ifc \txfm1\()_\txfm2, dct_dct idct_dc 16, 16, 2 .endif +.ifc \txfm1, identity + adr x9, inv_txfm_horz_identity_16x8_neon +.else + adr x9, inv_txfm_horz_16x8_neon adr x4, inv_\txfm1\()_8x16_neon +.endif adr x5, inv_\txfm2\()_8x16_neon mov x13, #\eob_half b inv_txfm_add_16x16_neon endfunc .endm @@ -1467,88 +1514,135 @@ def_fn_16x16 flipadst, dct, 36 def_fn_16x16 flipadst, adst, 36 def_fn_16x16 flipadst, flipadst, 36 def_fn_16x16 identity, dct, 8 -function inv_txfm_add_16x4_neon +.macro def_fn_416_base variant +function inv_txfm_\variant\()add_16x4_neon mov x15, x30 movi v4.8h, #0 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - ld1 {v\i\().4h}, [x2] +.ifc \variant, identity_ +.irp i, v16.4h, v17.4h, v18.4h, v19.4h + ld1 {\i}, [x2] + st1 {v4.4h}, [x2], #8 +.endr +.irp i, v16.d, v17.d, v18.d, v19.d + ld1 {\i}[1], [x2] + st1 {v4.4h}, [x2], #8 +.endr + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 +.irp i, v20.4h, v21.4h, v22.4h, v23.4h + ld1 {\i}, [x2] + st1 {v4.4h}, [x2], #8 +.endr +.irp i, v20.d, v21.d, v22.d, v23.d + ld1 {\i}[1], [x2] + st1 {v4.4h}, [x2], #8 +.endr + + identity_8x16_shift1 v0.h[0] +.else +.irp i, v16.4h, v17.4h, v18.4h, v19.4h, v20.4h, v21.4h, v22.4h, v23.4h, v24.4h, v25.4h, v26.4h, v27.4h, v28.4h, v29.4h, v30.4h, v31.4h + ld1 {\i}, [x2] st1 {v4.4h}, [x2], #8 .endr blr x4 ins v16.d[1], v20.d[0] ins v17.d[1], v21.d[0] ins v18.d[1], v22.d[0] ins v19.d[1], v23.d[0] -.irp i, 16, 17, 18, 19 - srshr v\i\().8h, v\i\().8h, #1 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + srshr \i, \i, #1 .endr +.endif transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 blr x5 mov x6, x0 load_add_store_8x4 x6, x7 +.ifc \variant, identity_ + mov v16.16b, v20.16b + mov v17.16b, v21.16b + mov v18.16b, v22.16b + mov v19.16b, v23.16b +.else ins v24.d[1], v28.d[0] ins v25.d[1], v29.d[0] ins v26.d[1], v30.d[0] ins v27.d[1], v31.d[0] srshr v16.8h, v24.8h, #1 srshr v17.8h, v25.8h, #1 srshr v18.8h, v26.8h, #1 srshr v19.8h, v27.8h, #1 +.endif transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 blr x5 add x6, x0, #8 load_add_store_8x4 x6, x7 br x15 endfunc -function inv_txfm_add_4x16_neon +function inv_txfm_\variant\()add_4x16_neon mov x15, x30 movi v2.8h, #0 mov x11, #32 cmp w3, w13 b.lt 1f add x6, x2, #16 -.irp i, 16, 17, 18, 19 - ld1 {v\i\().8h}, [x6] +.ifc \variant, identity_ +.irp i, v24.8h, v25.8h, v26.8h, v27.8h + ld1 {\i}, [x6] + st1 {v2.8h}, [x6], x11 +.endr + mov w16, #(5793-4096)*8 + dup v0.4h, w16 + identity_8x4_shift1 v24, v25, v26, v27, v0.h[0] +.else +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + ld1 {\i}, [x6] st1 {v2.8h}, [x6], x11 .endr blr x4 srshr v24.8h, v16.8h, #1 srshr v25.8h, v17.8h, #1 srshr v26.8h, v18.8h, #1 srshr v27.8h, v19.8h, #1 +.endif transpose_4x8h v24, v25, v26, v27, v4, v5, v6, v7 ins v28.d[0], v24.d[1] ins v29.d[0], v25.d[1] ins v30.d[0], v26.d[1] ins v31.d[0], v27.d[1] b 2f 1: -.irp i, 24, 25, 26, 27, 28, 29, 30, 31 - movi v\i\().4h, #0 +.irp i, v24.4h, v25.4h, v26.4h, v27.4h, v28.4h, v29.4h, v30.4h, v31.4h + movi \i, #0 .endr 2: movi v2.8h, #0 -.irp i, 16, 17, 18, 19 - ld1 {v\i\().8h}, [x2] +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + ld1 {\i}, [x2] st1 {v2.8h}, [x2], x11 .endr +.ifc \variant, identity_ + mov w16, #(5793-4096)*8 + dup v0.4h, w16 + identity_8x4_shift1 v16, v17, v18, v19, v0.h[0] +.else blr x4 -.irp i, 16, 17, 18, 19 - srshr v\i\().8h, v\i\().8h, #1 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + srshr \i, \i, #1 .endr +.endif transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7 ins v20.d[0], v16.d[1] ins v21.d[0], v17.d[1] ins v22.d[0], v18.d[1] ins v23.d[0], v19.d[1] @@ -1557,10 +1651,14 @@ load_add_store_4x16 x0, x6 br x15 endfunc +.endm + +def_fn_416_base +def_fn_416_base identity_ .macro def_fn_416 w, h, txfm1, txfm2, eob_half function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_neon, export=1 .ifc \txfm1\()_\txfm2, dct_dct idct_dc \w, \h, 1 @@ -1571,11 +1669,15 @@ mov w13, #\eob_half .else adr x4, inv_\txfm1\()_4x\w\()_neon adr x5, inv_\txfm2\()_8x\h\()_neon .endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else b inv_txfm_add_\w\()x\h\()_neon +.endif endfunc .endm .macro def_fns_416 w, h def_fn_416 \w, \h, dct, dct, 29 @@ -1598,43 +1700,61 @@ def_fns_416 4, 16 def_fns_416 16, 4 -function inv_txfm_add_16x8_neon +.macro def_fn_816_base variant +function inv_txfm_\variant\()add_16x8_neon mov x15, x30 movi v4.8h, #0 mov w16, #2896*8 dup v0.4h, w16 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - ld1 {v\i\().8h}, [x2] +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x2] st1 {v4.8h}, [x2], #16 .endr scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 +.ifc \variant, identity_ + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 + identity_8x16_shift1 v0.h[0] +.else blr x4 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23 - srshr v\i\().8h, v\i\().8h, #1 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + srshr \i, \i, #1 .endr +.endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v2, v3 blr x5 mov x6, x0 load_add_store_8x8 x6, x7 +.ifc \variant, identity_ + mov v16.16b, v24.16b + mov v17.16b, v25.16b + mov v18.16b, v26.16b + mov v19.16b, v27.16b + mov v20.16b, v28.16b + mov v21.16b, v29.16b + mov v22.16b, v30.16b + mov v23.16b, v31.16b +.else srshr v16.8h, v24.8h, #1 srshr v17.8h, v25.8h, #1 srshr v18.8h, v26.8h, #1 srshr v19.8h, v27.8h, #1 srshr v20.8h, v28.8h, #1 srshr v21.8h, v29.8h, #1 srshr v22.8h, v30.8h, #1 srshr v23.8h, v31.8h, #1 +.endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v2, v3 blr x5 @@ -1642,23 +1762,31 @@ load_add_store_8x8 x0, x7 br x15 endfunc -function inv_txfm_add_8x16_neon +function inv_txfm_\variant\()add_8x16_neon mov x15, x30 movi v4.8h, #0 mov w16, #2896*8 dup v0.4h, w16 mov x11, #32 cmp w3, w13 b.lt 1f add x6, x2, #16 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23 - ld1 {v\i\().8h}, [x6] +.ifc \variant, identity_ +.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x6] + st1 {v4.8h}, [x6], x11 +.endr + scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 + // The identity shl #1 and downshift srshr #1 cancel out +.else +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + ld1 {\i}, [x6] st1 {v4.8h}, [x6], x11 .endr scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 blr x4 @@ -1668,43 +1796,52 @@ srshr v27.8h, v19.8h, #1 srshr v28.8h, v20.8h, #1 srshr v29.8h, v21.8h, #1 srshr v30.8h, v22.8h, #1 srshr v31.8h, v23.8h, #1 +.endif transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v2, v3 b 2f 1: -.irp i, 24, 25, 26, 27, 28, 29, 30, 31 - movi v\i\().8h, #0 +.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + movi \i, #0 .endr 2: movi v4.8h, #0 mov w16, #2896*8 dup v0.4h, w16 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23 - ld1 {v\i\().8h}, [x2] +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + ld1 {\i}, [x2] st1 {v4.8h}, [x2], x11 .endr scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 +.ifc \variant, identity_ + // The identity shl #1 and downshift srshr #1 cancel out +.else blr x4 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23 - srshr v\i\().8h, v\i\().8h, #1 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + srshr \i, \i, #1 .endr +.endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v2, v3 blr x5 load_add_store_8x16 x0, x6 br x15 endfunc +.endm + +def_fn_816_base +def_fn_816_base identity_ .macro def_fn_816 w, h, txfm1, txfm2, eob_half function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_neon, export=1 .ifc \txfm1\()_\txfm2, dct_dct idct_dc \w, \h, 1 @@ -1712,11 +1849,15 @@ adr x4, inv_\txfm1\()_8x\w\()_neon adr x5, inv_\txfm2\()_8x\h\()_neon .if \w == 8 mov x13, #\eob_half .endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else b inv_txfm_add_\w\()x\h\()_neon +.endif endfunc .endm .macro def_fns_816 w, h def_fn_816 \w, \h, dct, dct, 43 @@ -2118,11 +2259,11 @@ .endm .macro def_identity_1632 w, h, wshort, hshort function inv_txfm_add_identity_identity_\w\()x\h\()_neon, export=1 mov w16, #2896*8 - mov w17, #2*5793 + mov w17, #2*(5793-4096)*8 dup v1.4h, w16 movi v0.8h, #0 mov v1.h[1], w17 movrel x13, eob_16x32\hshort @@ -2138,16 +2279,15 @@ .endr scale_input .8h, v1.h[0], v16, v17, v18, v19, v20, v21, v22, v23 .if \w == 16 // 16x32 - scale_wide .8h, v1.h[1], v16, v17, v18, v19, v20, v21, v22, v23 - shift_8_regs srshr, 1 + identity_8x8_shift1 v1.h[1] .else // 32x16 - shift_8_regs shl, 1 - scale_wide .8h, v1.h[1], v16, v17, v18, v19, v20, v21, v22, v23 + shift_8_regs sqshl, 1 + identity_8x8 v1.h[1] .endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 .if \w == 16 diff --git a/third_party/dav1d/src/arm/64/loopfilter.S b/third_party/dav1d/src/arm/64/loopfilter.S --- a/third_party/dav1d/src/arm/64/loopfilter.S +++ b/third_party/dav1d/src/arm/64/loopfilter.S @@ -149,12 +149,12 @@ saddw v2.8h, v2.8h, v4.8b saddw2 v3.8h, v3.8h, v4.16b movi v7.16b, #3 sqxtn v2.8b, v2.8h // f sqxtn2 v2.16b, v3.8h - sqadd v4.16b, v6.16b, v2.16b // imin(f + 4, 128) - sqadd v5.16b, v7.16b, v2.16b // imin(f + 3, 128) + sqadd v4.16b, v6.16b, v2.16b // imin(f + 4, 127) + sqadd v5.16b, v7.16b, v2.16b // imin(f + 3, 127) sshr v4.16b, v4.16b, #3 // f1 sshr v5.16b, v5.16b, #3 // f2 uxtl v2.8h, v23.8b // p0 uxtl2 v3.8h, v23.16b uxtl v6.8h, v24.8b // q0 @@ -979,17 +979,17 @@ st1 {v25.s}[3], [x0], x1 add x0, x0, #2 br x15 endfunc -// void dav1d_lpf_v_sb_y_neon(pixel *dst, const ptrdiff_t stride, -// const uint32_t *const vmask, -// const uint8_t (*l)[4], ptrdiff_t b4_stride, -// const Av1FilterLUT *lut, const int w) +// void dav1d_lpf_v_sb_y_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint32_t *const vmask, +// const uint8_t (*l)[4], ptrdiff_t b4_stride, +// const Av1FilterLUT *lut, const int w) .macro lpf_func dir, type -function lpf_\dir\()_sb_\type\()_neon, export=1 +function lpf_\dir\()_sb_\type\()_8bpc_neon, export=1 mov x11, x30 stp d8, d9, [sp, #-0x40]! stp d10, d11, [sp, #0x10] stp d12, d13, [sp, #0x20] stp d14, d15, [sp, #0x30] diff --git a/third_party/dav1d/src/arm/64/loopfilter16.S b/third_party/dav1d/src/arm/64/loopfilter16.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/loopfilter16.S @@ -0,0 +1,907 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro loop_filter wd +function lpf_8_wd\wd\()_neon + uabd v0.8h, v22.8h, v23.8h // abs(p1 - p0) + uabd v1.8h, v25.8h, v24.8h // abs(q1 - q0) + uabd v2.8h, v23.8h, v24.8h // abs(p0 - q0) + uabd v3.8h, v22.8h, v25.8h // abs(p1 - q1) +.if \wd >= 6 + uabd v4.8h, v21.8h, v22.8h // abs(p2 - p1) + uabd v5.8h, v26.8h, v25.8h // abs(q2 - q1) +.endif +.if \wd >= 8 + uabd v6.8h, v20.8h, v21.8h // abs(p3 - p2) + uabd v7.8h, v27.8h, v26.8h // abs(q3 - q3) +.endif +.if \wd >= 6 + umax v4.8h, v4.8h, v5.8h +.endif + uqadd v2.8h, v2.8h, v2.8h // abs(p0 - q0) * 2 +.if \wd >= 8 + umax v6.8h, v6.8h, v7.8h +.endif + ushr v3.8h, v3.8h, #1 +.if \wd >= 8 + umax v4.8h, v4.8h, v6.8h +.endif +.if \wd >= 6 + and v4.16b, v4.16b, v14.16b +.endif + umax v0.8h, v0.8h, v1.8h // max(abs(p1 - p0), abs(q1 - q0)) + uqadd v2.8h, v2.8h, v3.8h // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 +.if \wd >= 6 + umax v4.8h, v0.8h, v4.8h + cmhs v1.8h, v11.8h, v4.8h // max(abs(p1 - p0), abs(q1 - q0), abs(), abs(), ...) <= I +.else + cmhs v1.8h, v11.8h, v0.8h // max(abs(p1 - p0), abs(q1 - q0)) <= I +.endif + cmhs v2.8h, v10.8h, v2.8h // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 <= E + and v1.16b, v1.16b, v2.16b // fm + and v1.16b, v1.16b, v13.16b // fm && wd >= 4 +.if \wd >= 6 + and v14.16b, v14.16b, v1.16b // fm && wd > 4 +.endif +.if \wd >= 16 + and v15.16b, v15.16b, v1.16b // fm && wd == 16 +.endif + + mov x16, v1.d[0] + mov x17, v1.d[1] + adds x16, x16, x17 + b.eq 9f // if (!fm || wd < 4) return; + +.if \wd >= 6 + movi v10.8h, #1 + uabd v2.8h, v21.8h, v23.8h // abs(p2 - p0) + uabd v3.8h, v22.8h, v23.8h // abs(p1 - p0) + uabd v4.8h, v25.8h, v24.8h // abs(q1 - q0) + uabd v5.8h, v26.8h, v24.8h // abs(q2 - q0) + dup v9.8h, w9 // bitdepth_min_8 +.if \wd >= 8 + uabd v6.8h, v20.8h, v23.8h // abs(p3 - p0) + uabd v7.8h, v27.8h, v24.8h // abs(q3 - q0) +.endif + umax v2.8h, v2.8h, v3.8h + umax v4.8h, v4.8h, v5.8h +.if \wd >= 8 + umax v6.8h, v6.8h, v7.8h +.endif + umax v2.8h, v2.8h, v4.8h + ushl v10.8h, v10.8h, v9.8h // F = 1 << bitdepth_min_8 +.if \wd >= 8 + umax v2.8h, v2.8h, v6.8h +.endif + +.if \wd == 16 + uabd v3.8h, v17.8h, v23.8h // abs(p6 - p0) + uabd v4.8h, v18.8h, v23.8h // abs(p5 - p0) + uabd v5.8h, v19.8h, v23.8h // abs(p4 - p0) +.endif + cmhs v2.8h, v10.8h, v2.8h // flat8in +.if \wd == 16 + uabd v6.8h, v28.8h, v24.8h // abs(q4 - q0) + uabd v7.8h, v29.8h, v24.8h // abs(q5 - q0) + uabd v8.8h, v30.8h, v24.8h // abs(q6 - q0) +.endif + and v14.16b, v2.16b, v14.16b // flat8in && fm && wd > 4 + bic v1.16b, v1.16b, v14.16b // fm && wd >= 4 && !flat8in +.if \wd == 16 + umax v3.8h, v3.8h, v4.8h + umax v5.8h, v5.8h, v6.8h +.endif + mov x16, v1.d[0] + mov x17, v1.d[1] +.if \wd == 16 + umax v7.8h, v7.8h, v8.8h + umax v3.8h, v3.8h, v5.8h + umax v3.8h, v3.8h, v7.8h + cmhs v3.8h, v10.8h, v3.8h // flat8out +.endif + adds x16, x16, x17 +.if \wd == 16 + and v15.16b, v15.16b, v3.16b // flat8out && fm && wd == 16 + and v15.16b, v15.16b, v14.16b // flat8out && flat8in && fm && wd == 16 + bic v14.16b, v14.16b, v15.16b // flat8in && fm && wd >= 4 && !flat8out +.endif + b.eq 1f // skip wd == 4 case +.endif + + dup v3.8h, w8 // bitdepth_max + sub v2.8h, v22.8h, v25.8h // p1 - q1 + ushr v3.8h, v3.8h, #1 // 128 << bitdepth_min_8 - 1 + cmhi v0.8h, v0.8h, v12.8h // hev + not v9.16b, v3.16b // - 128 * (1 << bitdepth_min_8) + smin v2.8h, v2.8h, v3.8h // iclip_diff(p1 - q1) + smax v2.8h, v2.8h, v9.8h // iclip_diff(p1 - q1) + and v4.16b, v2.16b, v0.16b // if (hev) iclip_diff(p1 - q1) + sub v2.8h, v24.8h, v23.8h + movi v5.8h, #3 + bic v0.16b, v1.16b, v0.16b // (fm && wd >= 4 && !hev) + mul v2.8h, v2.8h, v5.8h + movi v6.8h, #4 + add v2.8h, v2.8h, v4.8h + smin v2.8h, v2.8h, v3.8h // f = iclip_diff() + movi v7.8h, #3 + smax v2.8h, v2.8h, v9.8h // f = iclip_diff() + sqadd v4.8h, v6.8h, v2.8h // f + 4 + sqadd v5.8h, v7.8h, v2.8h // f + 3 + smin v4.8h, v4.8h, v3.8h // imin(f + 4, 128 << bitdepth_min_8 - 1) + smin v5.8h, v5.8h, v3.8h // imin(f + 3, 128 << bitdepth_min_8 - 1) + sshr v4.8h, v4.8h, #3 // f1 + sshr v5.8h, v5.8h, #3 // f2 + movi v9.8h, #0 + dup v3.8h, w8 // bitdepth_max + sqadd v2.8h, v23.8h, v5.8h // p0 + f2 + sqsub v6.8h, v24.8h, v4.8h // q0 - f1 + srshr v4.8h, v4.8h, #1 // (f1 + 1) >> 1 + smin v2.8h, v2.8h, v3.8h // out p0 = iclip_pixel() + smin v6.8h, v6.8h, v3.8h // out q0 = iclip_pixel() + smax v2.8h, v2.8h, v9.8h // out p0 = iclip_pixel() + smax v6.8h, v6.8h, v9.8h // out q0 = iclip_pixel() + bit v23.16b, v2.16b, v1.16b // if (fm && wd >= 4) + bit v24.16b, v6.16b, v1.16b // if (fm && wd >= 4) + sqadd v2.8h, v22.8h, v4.8h // p1 + f + sqsub v6.8h, v25.8h, v4.8h // q1 - f + smin v2.8h, v2.8h, v3.8h // out p1 = iclip_pixel() + smin v6.8h, v6.8h, v3.8h // out q1 = iclip_pixel() + smax v2.8h, v2.8h, v9.8h // out p1 = iclip_pixel() + smax v6.8h, v6.8h, v9.8h // out q1 = iclip_pixel() + bit v22.16b, v2.16b, v0.16b // if (fm && wd >= 4 && !hev) + bit v25.16b, v6.16b, v0.16b // if (fm && wd >= 4 && !hev) +1: + +.if \wd == 6 + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 + b.eq 2f // skip if there's no flat8in + + add v0.8h, v21.8h, v21.8h // p2 * 2 + add v2.8h, v21.8h, v22.8h // p2 + p1 + add v4.8h, v22.8h, v23.8h // p1 + p0 + add v6.8h, v23.8h, v24.8h // p0 + q0 + add v8.8h, v0.8h, v2.8h + add v10.8h, v4.8h, v6.8h + add v12.8h, v24.8h, v25.8h // q0 + q1 + add v8.8h, v8.8h, v10.8h + sub v12.8h, v12.8h, v0.8h + add v10.8h, v25.8h, v26.8h // q1 + q2 + urshr v0.8h, v8.8h, #3 // out p1 + + add v8.8h, v8.8h, v12.8h + sub v10.8h, v10.8h, v2.8h + add v12.8h, v26.8h, v26.8h // q2 + q2 + urshr v1.8h, v8.8h, #3 // out p0 + + add v8.8h, v8.8h, v10.8h + sub v12.8h, v12.8h, v4.8h + urshr v2.8h, v8.8h, #3 // out q0 + + bit v22.16b, v0.16b, v14.16b // p1 if (flat8in) + add v8.8h, v8.8h, v12.8h + bit v23.16b, v1.16b, v14.16b // p0 if (flat8in) + urshr v3.8h, v8.8h, #3 // out q1 + bit v24.16b, v2.16b, v14.16b // q0 if (flat8in) + bit v25.16b, v3.16b, v14.16b // q1 if (flat8in) +.elseif \wd >= 8 + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 +.if \wd == 8 + b.eq 8f // skip if there's no flat8in +.else + b.eq 2f // skip if there's no flat8in +.endif + + add v0.8h, v20.8h, v21.8h // p3 + p2 + add v2.8h, v22.8h, v25.8h // p1 + q1 + add v4.8h, v20.8h, v22.8h // p3 + p1 + add v6.8h, v23.8h, v26.8h // p0 + q2 + add v8.8h, v0.8h, v0.8h // 2 * (p3 + p2) + add v9.8h, v23.8h, v24.8h // p0 + q0 + add v8.8h, v8.8h, v4.8h // + p3 + p1 + sub v2.8h, v2.8h, v0.8h // p1 + q1 - p3 - p2 + add v8.8h, v8.8h, v9.8h // + p0 + q0 + sub v6.8h, v6.8h, v4.8h // p0 + q2 - p3 - p1 + urshr v10.8h, v8.8h, #3 // out p2 + + add v8.8h, v8.8h, v2.8h + add v0.8h, v20.8h, v23.8h // p3 + p0 + add v2.8h, v24.8h, v27.8h // q0 + q3 + urshr v11.8h, v8.8h, #3 // out p1 + + add v8.8h, v8.8h, v6.8h + sub v2.8h, v2.8h, v0.8h // q0 + q3 - p3 - p0 + add v4.8h, v21.8h, v24.8h // p2 + q0 + add v6.8h, v25.8h, v27.8h // q1 + q3 + urshr v12.8h, v8.8h, #3 // out p0 + + add v8.8h, v8.8h, v2.8h + sub v6.8h, v6.8h, v4.8h // q1 + q3 - p2 - q0 + add v0.8h, v22.8h, v25.8h // p1 + q1 + add v2.8h, v26.8h, v27.8h // q2 + q3 + urshr v13.8h, v8.8h, #3 // out q0 + + add v8.8h, v8.8h, v6.8h + sub v2.8h, v2.8h, v0.8h // q2 + q3 - p1 - q1 + urshr v0.8h, v8.8h, #3 // out q1 + + add v8.8h, v8.8h, v2.8h + + bit v21.16b, v10.16b, v14.16b + bit v22.16b, v11.16b, v14.16b + bit v23.16b, v12.16b, v14.16b + urshr v1.8h, v8.8h, #3 // out q2 + bit v24.16b, v13.16b, v14.16b + bit v25.16b, v0.16b, v14.16b + bit v26.16b, v1.16b, v14.16b +.endif +2: +.if \wd == 16 + mov x16, v15.d[0] + mov x17, v15.d[1] + adds x16, x16, x17 + b.ne 1f // check if flat8out is needed + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 + b.eq 8f // if there was no flat8in, just write the inner 4 pixels + b 7f // if flat8in was used, write the inner 6 pixels +1: + + add v2.8h, v17.8h, v17.8h // p6 + p6 + add v4.8h, v17.8h, v18.8h // p6 + p5 + add v6.8h, v17.8h, v19.8h // p6 + p4 + add v8.8h, v17.8h, v20.8h // p6 + p3 + add v12.8h, v2.8h, v4.8h + add v10.8h, v6.8h, v8.8h + add v6.8h, v17.8h, v21.8h // p6 + p2 + add v12.8h, v12.8h, v10.8h + add v8.8h, v17.8h, v22.8h // p6 + p1 + add v10.8h, v18.8h, v23.8h // p5 + p0 + add v6.8h, v6.8h, v8.8h + add v8.8h, v19.8h, v24.8h // p4 + q0 + add v12.8h, v12.8h, v6.8h + add v10.8h, v10.8h, v8.8h + add v6.8h, v20.8h, v25.8h // p3 + q1 + add v12.8h, v12.8h, v10.8h + sub v6.8h, v6.8h, v2.8h + add v2.8h, v21.8h, v26.8h // p2 + q2 + urshr v0.8h, v12.8h, #4 // out p5 + add v12.8h, v12.8h, v6.8h // - (p6 + p6) + (p3 + q1) + sub v2.8h, v2.8h, v4.8h + add v4.8h, v22.8h, v27.8h // p1 + q3 + add v6.8h, v17.8h, v19.8h // p6 + p4 + urshr v1.8h, v12.8h, #4 // out p4 + add v12.8h, v12.8h, v2.8h // - (p6 + p5) + (p2 + q2) + sub v4.8h, v4.8h, v6.8h + add v6.8h, v23.8h, v28.8h // p0 + q4 + add v8.8h, v17.8h, v20.8h // p6 + p3 + urshr v2.8h, v12.8h, #4 // out p3 + add v12.8h, v12.8h, v4.8h // - (p6 + p4) + (p1 + q3) + sub v6.8h, v6.8h, v8.8h + add v8.8h, v24.8h, v29.8h // q0 + q5 + add v4.8h, v17.8h, v21.8h // p6 + p2 + urshr v3.8h, v12.8h, #4 // out p2 + add v12.8h, v12.8h, v6.8h // - (p6 + p3) + (p0 + q4) + sub v8.8h, v8.8h, v4.8h + add v6.8h, v25.8h, v30.8h // q1 + q6 + add v10.8h, v17.8h, v22.8h // p6 + p1 + urshr v4.8h, v12.8h, #4 // out p1 + add v12.8h, v12.8h, v8.8h // - (p6 + p2) + (q0 + q5) + sub v6.8h, v6.8h, v10.8h + add v8.8h, v26.8h, v30.8h // q2 + q6 + bif v0.16b, v18.16b, v15.16b // out p5 + add v10.8h, v18.8h, v23.8h // p5 + p0 + urshr v5.8h, v12.8h, #4 // out p0 + add v12.8h, v12.8h, v6.8h // - (p6 + p1) + (q1 + q6) + sub v8.8h, v8.8h, v10.8h + add v10.8h, v27.8h, v30.8h // q3 + q6 + bif v1.16b, v19.16b, v15.16b // out p4 + add v18.8h, v19.8h, v24.8h // p4 + q0 + urshr v6.8h, v12.8h, #4 // out q0 + add v12.8h, v12.8h, v8.8h // - (p5 + p0) + (q2 + q6) + sub v10.8h, v10.8h, v18.8h + add v8.8h, v28.8h, v30.8h // q4 + q6 + bif v2.16b, v20.16b, v15.16b // out p3 + add v18.8h, v20.8h, v25.8h // p3 + q1 + urshr v7.8h, v12.8h, #4 // out q1 + add v12.8h, v12.8h, v10.8h // - (p4 + q0) + (q3 + q6) + sub v18.8h, v8.8h, v18.8h + add v10.8h, v29.8h, v30.8h // q5 + q6 + bif v3.16b, v21.16b, v15.16b // out p2 + add v20.8h, v21.8h, v26.8h // p2 + q2 + urshr v8.8h, v12.8h, #4 // out q2 + add v12.8h, v12.8h, v18.8h // - (p3 + q1) + (q4 + q6) + sub v10.8h, v10.8h, v20.8h + add v18.8h, v30.8h, v30.8h // q6 + q6 + bif v4.16b, v22.16b, v15.16b // out p1 + add v20.8h, v22.8h, v27.8h // p1 + q3 + urshr v9.8h, v12.8h, #4 // out q3 + add v12.8h, v12.8h, v10.8h // - (p2 + q2) + (q5 + q6) + sub v18.8h, v18.8h, v20.8h + bif v5.16b, v23.16b, v15.16b // out p0 + urshr v10.8h, v12.8h, #4 // out q4 + add v12.8h, v12.8h, v18.8h // - (p1 + q3) + (q6 + q6) + urshr v11.8h, v12.8h, #4 // out q5 + bif v6.16b, v24.16b, v15.16b // out q0 + bif v7.16b, v25.16b, v15.16b // out q1 + bif v8.16b, v26.16b, v15.16b // out q2 + bif v9.16b, v27.16b, v15.16b // out q3 + bif v10.16b, v28.16b, v15.16b // out q4 + bif v11.16b, v29.16b, v15.16b // out q5 +.endif + + ret +.if \wd == 16 +7: + // Return to a shorter epilogue, writing only the inner 6 pixels + br x13 +.endif +.if \wd >= 8 +8: + // Return to a shorter epilogue, writing only the inner 4 pixels + br x14 +.endif +9: + // Return directly without writing back any pixels + br x15 +endfunc +.endm + +loop_filter 16 +loop_filter 8 +loop_filter 6 +loop_filter 4 + +.macro lpf_8_wd16 + adr x13, 7f + adr x14, 8f + bl lpf_8_wd16_neon +.endm + +.macro lpf_8_wd8 + adr x14, 8f + bl lpf_8_wd8_neon +.endm + +.macro lpf_8_wd6 + bl lpf_8_wd6_neon +.endm + +.macro lpf_8_wd4 + bl lpf_8_wd4_neon +.endm + +function lpf_v_4_8_neon + mov x15, x30 + sub x16, x0, x1, lsl #1 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + + lpf_8_wd4 + + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + br x15 +endfunc + +function lpf_h_4_8_neon + mov x15, x30 + sub x16, x0, #4 + add x0, x16, x1, lsl #2 + ld1 {v22.d}[0], [x16], x1 + ld1 {v22.d}[1], [x0], x1 + ld1 {v23.d}[0], [x16], x1 + ld1 {v23.d}[1], [x0], x1 + ld1 {v24.d}[0], [x16], x1 + ld1 {v24.d}[1], [x0], x1 + ld1 {v25.d}[0], [x16], x1 + ld1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_8_wd4 + + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + br x15 +endfunc + +function lpf_v_6_8_neon + mov x15, x30 + sub x16, x0, x1, lsl #1 + sub x16, x16, x1 + ld1 {v21.8h}, [x16], x1 // p2 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v25.8h}, [x0], x1 // q1 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v26.8h}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + + lpf_8_wd6 + + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + br x15 +endfunc + +function lpf_h_6_8_neon + mov x15, x30 + sub x16, x0, #8 + add x0, x16, x1, lsl #2 + ld1 {v20.8h}, [x16], x1 + ld1 {v24.8h}, [x0], x1 + ld1 {v21.8h}, [x16], x1 + ld1 {v25.8h}, [x0], x1 + ld1 {v22.8h}, [x16], x1 + ld1 {v26.8h}, [x0], x1 + ld1 {v23.8h}, [x16], x1 + ld1 {v27.8h}, [x0], x1 + add x0, x0, #8 + + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_8_wd6 + + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + br x15 +endfunc + +function lpf_v_8_8_neon + mov x15, x30 + sub x16, x0, x1, lsl #2 + ld1 {v20.8h}, [x16], x1 // p3 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v21.8h}, [x16], x1 // p2 + ld1 {v25.8h}, [x0], x1 // q1 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v26.8h}, [x0], x1 // q2 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v27.8h}, [x0], x1 // q3 + sub x0, x0, x1, lsl #2 + + lpf_8_wd8 + + sub x16, x0, x1, lsl #1 + sub x16, x16, x1 + st1 {v21.8h}, [x16], x1 // p2 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v25.8h}, [x0], x1 // q1 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v26.8h}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + br x15 + +8: + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + br x15 +endfunc + +function lpf_h_8_8_neon + mov x15, x30 + sub x16, x0, #8 + add x0, x16, x1, lsl #2 + ld1 {v20.8h}, [x16], x1 + ld1 {v24.8h}, [x0], x1 + ld1 {v21.8h}, [x16], x1 + ld1 {v25.8h}, [x0], x1 + ld1 {v22.8h}, [x16], x1 + ld1 {v26.8h}, [x0], x1 + ld1 {v23.8h}, [x16], x1 + ld1 {v27.8h}, [x0], x1 + add x0, x0, #8 + + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_8_wd8 + + sub x16, x0, x1, lsl #3 + sub x16, x16, #8 + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v20.8h}, [x16], x1 + st1 {v24.8h}, [x0], x1 + st1 {v21.8h}, [x16], x1 + st1 {v25.8h}, [x0], x1 + st1 {v22.8h}, [x16], x1 + st1 {v26.8h}, [x0], x1 + st1 {v23.8h}, [x16], x1 + st1 {v27.8h}, [x0], x1 + add x0, x0, #8 + br x15 +8: + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + br x15 +endfunc + +function lpf_v_16_8_neon + mov x15, x30 + + sub x16, x0, x1, lsl #3 + add x16, x16, x1 + ld1 {v17.8h}, [x16], x1 // p6 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v18.8h}, [x16], x1 // p5 + ld1 {v25.8h}, [x0], x1 // q1 + ld1 {v19.8h}, [x16], x1 // p4 + ld1 {v26.8h}, [x0], x1 // q2 + ld1 {v20.8h}, [x16], x1 // p3 + ld1 {v27.8h}, [x0], x1 // q3 + ld1 {v21.8h}, [x16], x1 // p2 + ld1 {v28.8h}, [x0], x1 // q4 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v29.8h}, [x0], x1 // q5 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v30.8h}, [x0], x1 // q6 + sub x0, x0, x1, lsl #3 + add x0, x0, x1 + + lpf_8_wd16 + + sub x16, x0, x1, lsl #2 + sub x16, x16, x1, lsl #1 + st1 {v0.8h}, [x16], x1 // p5 + st1 {v6.8h}, [x0], x1 // q0 + st1 {v1.8h}, [x16], x1 // p4 + st1 {v7.8h}, [x0], x1 // q1 + st1 {v2.8h}, [x16], x1 // p3 + st1 {v8.8h}, [x0], x1 // q2 + st1 {v3.8h}, [x16], x1 // p2 + st1 {v9.8h}, [x0], x1 // q3 + st1 {v4.8h}, [x16], x1 // p1 + st1 {v10.8h}, [x0], x1 // q4 + st1 {v5.8h}, [x16], x1 // p0 + st1 {v11.8h}, [x0], x1 // q5 + sub x0, x0, x1, lsl #2 + sub x0, x0, x1, lsl #1 + br x15 +7: + sub x16, x0, x1 + sub x16, x16, x1, lsl #1 + st1 {v21.8h}, [x16], x1 // p2 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v25.8h}, [x0], x1 // q1 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v26.8h}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + br x15 + +8: + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + br x15 +endfunc + +function lpf_h_16_8_neon + mov x15, x30 + sub x16, x0, #16 + ld1 {v16.8h}, [x16], x1 + ld1 {v24.8h}, [x0], x1 + ld1 {v17.8h}, [x16], x1 + ld1 {v25.8h}, [x0], x1 + ld1 {v18.8h}, [x16], x1 + ld1 {v26.8h}, [x0], x1 + ld1 {v19.8h}, [x16], x1 + ld1 {v27.8h}, [x0], x1 + ld1 {v20.8h}, [x16], x1 + ld1 {v28.8h}, [x0], x1 + ld1 {v21.8h}, [x16], x1 + ld1 {v29.8h}, [x0], x1 + ld1 {v22.8h}, [x16], x1 + ld1 {v30.8h}, [x0], x1 + ld1 {v23.8h}, [x16], x1 + ld1 {v31.8h}, [x0], x1 + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v0, v1 + transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v0, v1 + + lpf_8_wd16 + + sub x0, x0, x1, lsl #3 + sub x16, x0, #16 + + transpose_8x8h v16, v17, v0, v1, v2, v3, v4, v5, v18, v19 + transpose_8x8h v6, v7, v8, v9, v10, v11, v30, v31, v18, v19 + + st1 {v16.8h}, [x16], x1 + st1 {v6.8h}, [x0], x1 + st1 {v17.8h}, [x16], x1 + st1 {v7.8h}, [x0], x1 + st1 {v0.8h}, [x16], x1 + st1 {v8.8h}, [x0], x1 + st1 {v1.8h}, [x16], x1 + st1 {v9.8h}, [x0], x1 + st1 {v2.8h}, [x16], x1 + st1 {v10.8h}, [x0], x1 + st1 {v3.8h}, [x16], x1 + st1 {v11.8h}, [x0], x1 + st1 {v4.8h}, [x16], x1 + st1 {v30.8h}, [x0], x1 + st1 {v5.8h}, [x16], x1 + st1 {v31.8h}, [x0], x1 + br x15 + +7: + sub x16, x0, x1, lsl #3 + sub x16, x16, #8 + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v20.8h}, [x16], x1 + st1 {v24.8h}, [x0], x1 + st1 {v21.8h}, [x16], x1 + st1 {v25.8h}, [x0], x1 + st1 {v22.8h}, [x16], x1 + st1 {v26.8h}, [x0], x1 + st1 {v23.8h}, [x16], x1 + st1 {v27.8h}, [x0], x1 + add x0, x0, #8 + br x15 +8: + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + br x15 +endfunc + +// void dav1d_lpf_v_sb_y_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint32_t *const vmask, +// const uint8_t (*l)[4], ptrdiff_t b4_stride, +// const Av1FilterLUT *lut, const int w, +// const int bitdepth_max) + +.macro lpf_func dir, type +function lpf_\dir\()_sb_\type\()_16bpc_neon, export=1 + mov x11, x30 + mov w8, w7 // bitdepth_max + clz w9, w8 + mov w10, #24 + sub w9, w10, w9 // bitdepth_min_8 + stp d8, d9, [sp, #-0x40]! + stp d10, d11, [sp, #0x10] + stp d12, d13, [sp, #0x20] + stp d14, d15, [sp, #0x30] + ldp w6, w7, [x2] // vmask[0], vmask[1] +.ifc \type, y + ldr w2, [x2, #8] // vmask[2] +.endif + add x5, x5, #128 // Move to sharp part of lut +.ifc \type, y + orr w7, w7, w2 // vmask[1] |= vmask[2] +.endif +.ifc \dir, v + sub x4, x3, x4, lsl #2 +.else + sub x3, x3, #4 + lsl x4, x4, #2 +.endif + orr w6, w6, w7 // vmask[0] |= vmask[1] + +1: + tst w6, #0x0f +.ifc \dir, v + ld1 {v0.8b}, [x4], #8 + ld1 {v1.8b}, [x3], #8 +.else + ld2 {v0.s,v1.s}[0], [x3], x4 + ld2 {v0.s,v1.s}[1], [x3], x4 +.endif + b.eq 7f // if (!(vm & bits)) continue; + + ld1r {v5.8b}, [x5] // sharp[0] + add x5, x5, #8 + movi v2.2s, #0xff + dup v13.2s, w6 // vmask[0] + dup v31.8h, w9 // bitdepth_min_8 + + and v0.8b, v0.8b, v2.8b // Keep only lowest byte in each 32 bit word + and v1.8b, v1.8b, v2.8b + cmtst v3.8b, v1.8b, v2.8b // Check for nonzero values in l[0][0] + movi v4.8b, #1 + ld1r {v6.8b}, [x5] // sharp[1] + sub x5, x5, #8 + bif v1.8b, v0.8b, v3.8b // if (!l[0][0]) L = l[offset][0] + mul v1.2s, v1.2s, v4.2s // L +.ifc \type, y + dup v15.2s, w2 // vmask[2] +.endif + cmtst v2.2s, v1.2s, v2.2s // L != 0 + dup v14.2s, w7 // vmask[1] + mov x16, v2.d[0] + cmp x16, #0 + b.eq 7f // if (!L) continue; + neg v5.8b, v5.8b // -sharp[0] + movrel x16, word_12 + ushr v12.8b, v1.8b, #4 // H + ld1 {v16.2s}, [x16] + sshl v3.8b, v1.8b, v5.8b // L >> sharp[0] +.ifc \type, y + cmtst v15.2s, v15.2s, v16.2s // if (vmask[2] & bits) +.endif + movi v7.8b, #2 + umin v3.8b, v3.8b, v6.8b // imin(L >> sharp[0], sharp[1]) + add v0.8b, v1.8b, v7.8b // L + 2 + umax v11.8b, v3.8b, v4.8b // imax(imin(), 1) = limit = I + add v0.8b, v0.8b, v0.8b // 2*(L + 2) + cmtst v14.2s, v14.2s, v16.2s // if (vmask[1] & bits) + uxtl v12.8h, v12.8b + add v10.8b, v0.8b, v11.8b // 2*(L + 2) + limit = E + cmtst v13.2s, v13.2s, v16.2s // if (vmask[0] & bits) + uxtl v11.8h, v11.8b + uxtl v10.8h, v10.8b + and v13.8b, v13.8b, v2.8b // vmask[0] &= L != 0 + sxtl v14.8h, v14.8b + sxtl v13.8h, v13.8b +.ifc \type, y + sxtl v15.8h, v15.8b +.endif + ushl v12.8h, v12.8h, v31.8h + ushl v11.8h, v11.8h, v31.8h + ushl v10.8h, v10.8h, v31.8h + +.ifc \type, y + tst w2, #0x0f + b.eq 2f + // wd16 + bl lpf_\dir\()_16_8_neon + b 8f +2: +.endif + tst w7, #0x0f + b.eq 3f +.ifc \type, y + // wd8 + bl lpf_\dir\()_8_8_neon +.else + // wd6 + bl lpf_\dir\()_6_8_neon +.endif + b 8f +3: + // wd4 + bl lpf_\dir\()_4_8_neon +.ifc \dir, h + b 8f +7: + // For dir h, the functions above increment x0. + // If the whole function is skipped, increment it here instead. + add x0, x0, x1, lsl #3 +.else +7: +.endif +8: + lsr w6, w6, #2 // vmask[0] >>= 2 + lsr w7, w7, #2 // vmask[1] >>= 2 +.ifc \type, y + lsr w2, w2, #2 // vmask[2] >>= 2 +.endif +.ifc \dir, v + add x0, x0, #16 +.else + // For dir h, x0 is returned incremented +.endif + cbnz w6, 1b + + ldp d14, d15, [sp, #0x30] + ldp d12, d13, [sp, #0x20] + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x40 + br x11 +endfunc +.endm + +lpf_func v, y +lpf_func h, y +lpf_func v, uv +lpf_func h, uv + +const word_12 + .word 1, 2 +endconst diff --git a/third_party/dav1d/src/arm/64/looprestoration.S b/third_party/dav1d/src/arm/64/looprestoration.S --- a/third_party/dav1d/src/arm/64/looprestoration.S +++ b/third_party/dav1d/src/arm/64/looprestoration.S @@ -26,15 +26,15 @@ */ #include "src/arm/asm.S" #include "util.S" -// void dav1d_wiener_filter_h_neon(int16_t *dst, const pixel (*left)[4], -// const pixel *src, ptrdiff_t stride, -// const int16_t fh[7], const intptr_t w, -// int h, enum LrEdgeFlags edges); -function wiener_filter_h_neon, export=1 +// void dav1d_wiener_filter_h_8bpc_neon(int16_t *dst, const pixel (*left)[4], +// const pixel *src, ptrdiff_t stride, +// const int16_t fh[7], const intptr_t w, +// int h, enum LrEdgeFlags edges); +function wiener_filter_h_8bpc_neon, export=1 mov w8, w5 ld1 {v0.8h}, [x4] mov w9, #(1 << 14) - (1 << 2) dup v30.8h, w9 movi v31.8h, #8, lsl #8 @@ -304,15 +304,15 @@ 0: ret .purgem filter endfunc -// void dav1d_wiener_filter_v_neon(pixel *dst, ptrdiff_t stride, -// const int16_t *mid, int w, int h, -// const int16_t fv[7], enum LrEdgeFlags edges, -// ptrdiff_t mid_stride); -function wiener_filter_v_neon, export=1 +// void dav1d_wiener_filter_v_8bpc_neon(pixel *dst, ptrdiff_t stride, +// const int16_t *mid, int w, int h, +// const int16_t fv[7], enum LrEdgeFlags edges, +// ptrdiff_t mid_stride); +function wiener_filter_v_8bpc_neon, export=1 mov w8, w4 ld1 {v0.8h}, [x5] movi v1.8h, #128 add v1.8h, v1.8h, v0.8h @@ -480,13 +480,13 @@ 0: ret .purgem filter endfunc -// void dav1d_copy_narrow_neon(pixel *dst, ptrdiff_t stride, -// const pixel *src, int w, int h); -function copy_narrow_neon, export=1 +// void dav1d_copy_narrow_8bpc_neon(pixel *dst, ptrdiff_t stride, +// const pixel *src, int w, int h); +function copy_narrow_8bpc_neon, export=1 adr x5, L(copy_narrow_tbl) ldrh w6, [x5, w3, uxtw #1] sub x5, x5, w6, uxth br x5 10: @@ -615,16 +615,18 @@ .hword L(copy_narrow_tbl) - 70b endfunc #define SUM_STRIDE (384+16) -// void dav1d_sgr_box3_h_neon(int32_t *sumsq, int16_t *sum, -// const pixel (*left)[4], -// const pixel *src, const ptrdiff_t stride, -// const int w, const int h, -// const enum LrEdgeFlags edges); -function sgr_box3_h_neon, export=1 +#include "looprestoration_tmpl.S" + +// void dav1d_sgr_box3_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_h_8bpc_neon, export=1 add w5, w5, #2 // w += 2 // Set up pointers for reading/writing alternate rows add x10, x0, #(4*SUM_STRIDE) // sumsq add x11, x1, #(2*SUM_STRIDE) // sum @@ -842,15 +844,15 @@ umull2 v2.8h, v0.16b, v0.16b umull v5.8h, v4.8b, v4.8b umull2 v6.8h, v4.16b, v4.16b add3 4 + subs w5, w5, #4 st1 {v3.4h}, [x1], #8 st1 {v7.4h}, [x11], #8 st1 {v26.4s}, [x0], #16 st1 {v28.4s}, [x10], #16 - subs w5, w5, #4 b.le 9f ext v0.16b, v0.16b, v0.16b, #4 ext v4.16b, v4.16b, v4.16b, #4 ext v1.16b, v1.16b, v2.16b, #8 ext v5.16b, v5.16b, v6.16b, #8 @@ -877,16 +879,16 @@ 0: ret .purgem add3 endfunc -// void dav1d_sgr_box5_h_neon(int32_t *sumsq, int16_t *sum, -// const pixel (*left)[4], -// const pixel *src, const ptrdiff_t stride, -// const int w, const int h, -// const enum LrEdgeFlags edges); -function sgr_box5_h_neon, export=1 +// void dav1d_sgr_box5_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_h_8bpc_neon, export=1 add w5, w5, #2 // w += 2 // Set up pointers for reading/writing alternate rows add x10, x0, #(4*SUM_STRIDE) // sumsq add x11, x1, #(2*SUM_STRIDE) // sum @@ -948,11 +950,11 @@ ext v0.16b, v1.16b, v0.16b, #13 ext v4.16b, v5.16b, v4.16b, #13 b 2f 0: // !LR_HAVE_LEFT, fill v1 with the leftmost byte - // and shift v0 to have 2x the first byte at the front. + // and shift v0 to have 3x the first byte at the front. dup v1.16b, v0.b[0] dup v5.16b, v4.b[0] // Move x3 back to account for the last 3 bytes we loaded before, // which we shifted out. sub x3, x3, #3 @@ -1112,15 +1114,15 @@ umull2 v2.8h, v0.16b, v0.16b umull v5.8h, v4.8b, v4.8b umull2 v6.8h, v4.16b, v4.16b add5 4 + subs w5, w5, #4 st1 {v3.4h}, [x1], #8 st1 {v7.4h}, [x11], #8 st1 {v26.4s}, [x0], #16 st1 {v28.4s}, [x10], #16 - subs w5, w5, #4 b.le 9f ext v0.16b, v0.16b, v0.16b, #4 ext v1.16b, v1.16b, v2.16b, #8 ext v4.16b, v4.16b, v4.16b, #4 ext v5.16b, v5.16b, v6.16b, #8 @@ -1145,841 +1147,6 @@ 0: ret .purgem add5 endfunc -// void dav1d_sgr_box3_v_neon(int32_t *sumsq, int16_t *sum, -// const int w, const int h, -// const enum LrEdgeFlags edges); -function sgr_box3_v_neon, export=1 - add w10, w3, #2 // Number of output rows to move back - mov w11, w3 // Number of input rows to move back - add w2, w2, #2 // Actual summed width - mov x7, #(4*SUM_STRIDE) // sumsq stride - mov x8, #(2*SUM_STRIDE) // sum stride - sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride - sub x1, x1, #(2*SUM_STRIDE) // sum -= stride - - tst w4, #4 // LR_HAVE_TOP - b.eq 0f - // If have top, read from row -2. - sub x5, x0, #(4*SUM_STRIDE) - sub x6, x1, #(2*SUM_STRIDE) - add w11, w11, #2 - b 1f -0: - // !LR_HAVE_TOP - // If we don't have top, read from row 0 even if - // we start writing to row -1. - add x5, x0, #(4*SUM_STRIDE) - add x6, x1, #(2*SUM_STRIDE) -1: - - tst w4, #8 // LR_HAVE_BOTTOM - b.eq 1f - // LR_HAVE_BOTTOM - add w3, w3, #2 // Sum all h+2 lines with the main loop - add w11, w11, #2 -1: - mov w9, w3 // Backup of h for next loops - -1: - // Start of horizontal loop; start one vertical filter slice. - // Start loading rows into v16-v21 and v24-v26 taking top - // padding into consideration. - tst w4, #4 // LR_HAVE_TOP - ld1 {v16.4s, v17.4s}, [x5], x7 - ld1 {v24.8h}, [x6], x8 - b.eq 2f - // LR_HAVE_TOP - ld1 {v18.4s, v19.4s}, [x5], x7 - ld1 {v25.8h}, [x6], x8 - ld1 {v20.4s, v21.4s}, [x5], x7 - ld1 {v26.8h}, [x6], x8 - b 3f -2: // !LR_HAVE_TOP - mov v18.16b, v16.16b - mov v19.16b, v17.16b - mov v25.16b, v24.16b - mov v20.16b, v16.16b - mov v21.16b, v17.16b - mov v26.16b, v24.16b - -3: - subs w3, w3, #1 -.macro add3 - add v16.4s, v16.4s, v18.4s - add v17.4s, v17.4s, v19.4s - add v24.8h, v24.8h, v25.8h - add v16.4s, v16.4s, v20.4s - add v17.4s, v17.4s, v21.4s - add v24.8h, v24.8h, v26.8h - st1 {v16.4s, v17.4s}, [x0], x7 - st1 {v24.8h}, [x1], x8 -.endm - add3 - mov v16.16b, v18.16b - mov v17.16b, v19.16b - mov v24.16b, v25.16b - mov v18.16b, v20.16b - mov v19.16b, v21.16b - mov v25.16b, v26.16b - b.le 4f - ld1 {v20.4s, v21.4s}, [x5], x7 - ld1 {v26.8h}, [x6], x8 - b 3b - -4: - tst w4, #8 // LR_HAVE_BOTTOM - b.ne 5f - // !LR_HAVE_BOTTOM - // Produce two more rows, extending the already loaded rows. - add3 - mov v16.16b, v18.16b - mov v17.16b, v19.16b - mov v24.16b, v25.16b - add3 - -5: // End of one vertical slice. - subs w2, w2, #8 - b.le 0f - // Move pointers back up to the top and loop horizontally. - // Input pointers - msub x5, x7, x11, x5 - msub x6, x8, x11, x6 - // Output pointers - msub x0, x7, x10, x0 - msub x1, x8, x10, x1 - add x0, x0, #32 - add x1, x1, #16 - add x5, x5, #32 - add x6, x6, #16 - mov w3, w9 - b 1b - -0: - ret -.purgem add3 -endfunc - -// void dav1d_sgr_box5_v_neon(int32_t *sumsq, int16_t *sum, -// const int w, const int h, -// const enum LrEdgeFlags edges); -function sgr_box5_v_neon, export=1 - add w10, w3, #2 // Number of output rows to move back - mov w11, w3 // Number of input rows to move back - add w2, w2, #8 // Actual summed width - mov x7, #(4*SUM_STRIDE) // sumsq stride - mov x8, #(2*SUM_STRIDE) // sum stride - sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride - sub x1, x1, #(2*SUM_STRIDE) // sum -= stride - - tst w4, #4 // LR_HAVE_TOP - b.eq 0f - // If have top, read from row -2. - sub x5, x0, #(4*SUM_STRIDE) - sub x6, x1, #(2*SUM_STRIDE) - add w11, w11, #2 - b 1f -0: - // !LR_HAVE_TOP - // If we don't have top, read from row 0 even if - // we start writing to row -1. - add x5, x0, #(4*SUM_STRIDE) - add x6, x1, #(2*SUM_STRIDE) -1: - - tst w4, #8 // LR_HAVE_BOTTOM - b.eq 0f - // LR_HAVE_BOTTOM - add w3, w3, #2 // Handle h+2 lines with the main loop - add w11, w11, #2 - b 1f -0: - // !LR_HAVE_BOTTOM - sub w3, w3, #1 // Handle h-1 lines with the main loop -1: - mov w9, w3 // Backup of h for next loops - -1: - // Start of horizontal loop; start one vertical filter slice. - // Start loading rows into v16-v25 and v26-v30 taking top - // padding into consideration. - tst w4, #4 // LR_HAVE_TOP - ld1 {v16.4s, v17.4s}, [x5], x7 - ld1 {v26.8h}, [x6], x8 - b.eq 2f - // LR_HAVE_TOP - ld1 {v20.4s, v21.4s}, [x5], x7 - ld1 {v28.8h}, [x6], x8 - mov v18.16b, v16.16b - mov v19.16b, v17.16b - mov v27.16b, v26.16b - ld1 {v22.4s, v23.4s}, [x5], x7 - ld1 {v29.8h}, [x6], x8 - b 3f -2: // !LR_HAVE_TOP - mov v18.16b, v16.16b - mov v19.16b, v17.16b - mov v27.16b, v26.16b - mov v20.16b, v16.16b - mov v21.16b, v17.16b - mov v28.16b, v26.16b - mov v22.16b, v16.16b - mov v23.16b, v17.16b - mov v29.16b, v26.16b - -3: - cbz w3, 4f - ld1 {v24.4s, v25.4s}, [x5], x7 - ld1 {v30.8h}, [x6], x8 - -3: - // Start of vertical loop - subs w3, w3, #2 -.macro add5 - add v16.4s, v16.4s, v18.4s - add v17.4s, v17.4s, v19.4s - add v26.8h, v26.8h, v27.8h - add v0.4s, v20.4s, v22.4s - add v1.4s, v21.4s, v23.4s - add v2.8h, v28.8h, v29.8h - add v16.4s, v16.4s, v24.4s - add v17.4s, v17.4s, v25.4s - add v26.8h, v26.8h, v30.8h - add v16.4s, v16.4s, v0.4s - add v17.4s, v17.4s, v1.4s - add v26.8h, v26.8h, v2.8h - st1 {v16.4s, v17.4s}, [x0], x7 - st1 {v26.8h}, [x1], x8 -.endm - add5 -.macro shift2 - mov v16.16b, v20.16b - mov v17.16b, v21.16b - mov v26.16b, v28.16b - mov v18.16b, v22.16b - mov v19.16b, v23.16b - mov v27.16b, v29.16b - mov v20.16b, v24.16b - mov v21.16b, v25.16b - mov v28.16b, v30.16b -.endm - shift2 - add x0, x0, x7 - add x1, x1, x8 - b.le 5f - ld1 {v22.4s, v23.4s}, [x5], x7 - ld1 {v29.8h}, [x6], x8 - ld1 {v24.4s, v25.4s}, [x5], x7 - ld1 {v30.8h}, [x6], x8 - b 3b - -4: - // h == 1, !LR_HAVE_BOTTOM. - // Pad the last row with the only content row, and add. - mov v24.16b, v22.16b - mov v25.16b, v23.16b - mov v30.16b, v29.16b - add5 - shift2 - add x0, x0, x7 - add x1, x1, x8 - add5 - b 6f - -5: - tst w4, #8 // LR_HAVE_BOTTOM - b.ne 6f - // !LR_HAVE_BOTTOM - cbnz w3, 5f - // The intended three edge rows left; output the one at h-2 and - // the past edge one at h. - ld1 {v22.4s, v23.4s}, [x5], x7 - ld1 {v29.8h}, [x6], x8 - // Pad the past-edge row from the last content row. - mov v24.16b, v22.16b - mov v25.16b, v23.16b - mov v30.16b, v29.16b - add5 - shift2 - add x0, x0, x7 - add x1, x1, x8 - // The last two rows are already padded properly here. - add5 - b 6f - -5: - // w3 == -1, two rows left, output one. - // Pad the last two rows from the mid one. - mov v22.16b, v20.16b - mov v23.16b, v21.16b - mov v29.16b, v28.16b - mov v24.16b, v20.16b - mov v25.16b, v21.16b - mov v30.16b, v28.16b - add5 - add x0, x0, x7 - add x1, x1, x8 - b 6f - -6: // End of one vertical slice. - subs w2, w2, #8 - b.le 0f - // Move pointers back up to the top and loop horizontally. - // Input pointers - msub x5, x7, x11, x5 - msub x6, x8, x11, x6 - // Output pointers - msub x0, x7, x10, x0 - msub x1, x8, x10, x1 - add x0, x0, #32 - add x1, x1, #16 - add x5, x5, #32 - add x6, x6, #16 - mov w3, w9 - b 1b - -0: - ret -.purgem add5 -endfunc - -// void dav1d_sgr_calc_ab1_neon(int32_t *a, int16_t *b, -// const int w, const int h, const int strength); -// void dav1d_sgr_calc_ab2_neon(int32_t *a, int16_t *b, -// const int w, const int h, const int strength); -function sgr_calc_ab1_neon, export=1 - add x3, x3, #2 // h += 2 - movi v31.4s, #9 // n - mov x5, #455 - mov x8, #SUM_STRIDE - b sgr_calc_ab_neon -endfunc - -function sgr_calc_ab2_neon, export=1 - add x3, x3, #3 // h += 3 - asr x3, x3, #1 // h /= 2 - movi v31.4s, #25 // n - mov x5, #164 - mov x8, #(2*SUM_STRIDE) -endfunc - -function sgr_calc_ab_neon - movrel x12, X(sgr_x_by_x) - ld1 {v16.16b, v17.16b, v18.16b}, [x12] - movi v19.16b, #5 - movi v20.8b, #55 // idx of last 5 - movi v21.8b, #72 // idx of last 4 - movi v22.8b, #101 // idx of last 3 - movi v23.8b, #169 // idx of last 2 - movi v24.8b, #254 // idx of last 1 - add x2, x2, #2 // w += 2 - add x7, x2, #7 - bic x7, x7, #7 // aligned w - sub x7, x8, x7 // increment between rows - movi v29.8h, #1, lsl #8 - dup v28.4s, w4 - dup v30.4s, w5 // one_by_x - sub x0, x0, #(4*(SUM_STRIDE)) - sub x1, x1, #(2*(SUM_STRIDE)) - mov x6, x2 // backup of w - sub v16.16b, v16.16b, v19.16b - sub v17.16b, v17.16b, v19.16b - sub v18.16b, v18.16b, v19.16b -1: - subs x2, x2, #8 - ld1 {v0.4s, v1.4s}, [x0] // a - ld1 {v2.8h}, [x1] // b - mul v0.4s, v0.4s, v31.4s // a * n - mul v1.4s, v1.4s, v31.4s // a * n - umull v3.4s, v2.4h, v2.4h // b * b - umull2 v4.4s, v2.8h, v2.8h // b * b - uqsub v0.4s, v0.4s, v3.4s // imax(a * n - b * b, 0) - uqsub v1.4s, v1.4s, v4.4s // imax(a * n - b * b, 0) - mul v0.4s, v0.4s, v28.4s // p * s - mul v1.4s, v1.4s, v28.4s // p * s - uqshrn v0.4h, v0.4s, #16 - uqshrn2 v0.8h, v1.4s, #16 - uqrshrn v0.8b, v0.8h, #4 // imin(z, 255) - - cmhi v25.8b, v0.8b, v20.8b // = -1 if sgr_x_by_x[v0] < 5 - cmhi v26.8b, v0.8b, v21.8b // = -1 if sgr_x_by_x[v0] < 4 - tbl v1.8b, {v16.16b, v17.16b, v18.16b}, v0.8b - cmhi v27.8b, v0.8b, v22.8b // = -1 if sgr_x_by_x[v0] < 3 - cmhi v5.8b, v0.8b, v23.8b // = -1 if sgr_x_by_x[v0] < 2 - add v25.8b, v25.8b, v26.8b - cmhi v6.8b, v0.8b, v24.8b // = -1 if sgr_x_by_x[v0] < 1 - add v27.8b, v27.8b, v5.8b - add v6.8b, v6.8b, v19.8b - add v25.8b, v25.8b, v27.8b - add v1.8b, v1.8b, v6.8b - add v1.8b, v1.8b, v25.8b - uxtl v1.8h, v1.8b // x - - umull v3.4s, v1.4h, v2.4h // x * BB[i] - umull2 v4.4s, v1.8h, v2.8h // x * BB[i] - mul v3.4s, v3.4s, v30.4s // x * BB[i] * sgr_one_by_x - mul v4.4s, v4.4s, v30.4s // x * BB[i] * sgr_one_by_x - srshr v3.4s, v3.4s, #12 // AA[i] - srshr v4.4s, v4.4s, #12 // AA[i] - sub v2.8h, v29.8h, v1.8h // 256 - x - - st1 {v3.4s, v4.4s}, [x0], #32 - st1 {v2.8h}, [x1], #16 - b.gt 1b - - subs x3, x3, #1 - b.le 0f - add x0, x0, x7, lsl #2 - add x1, x1, x7, lsl #1 - mov x2, x6 - b 1b -0: - ret -endfunc - -#define FILTER_OUT_STRIDE 384 - -// void dav1d_sgr_finish_filter1_neon(coef *tmp, -// const pixel *src, const ptrdiff_t stride, -// const int32_t *a, const int16_t *b, -// const int w, const int h); -function sgr_finish_filter1_neon, export=1 - sub x7, x3, #(4*SUM_STRIDE) - add x8, x3, #(4*SUM_STRIDE) - sub x9, x4, #(2*SUM_STRIDE) - add x10, x4, #(2*SUM_STRIDE) - mov x11, #SUM_STRIDE - mov x12, #FILTER_OUT_STRIDE - add x13, x5, #7 - bic x13, x13, #7 // Aligned width - sub x2, x2, x13 - sub x12, x12, x13 - sub x11, x11, x13 - sub x11, x11, #4 // We read 4 extra elements from a - sub x14, x11, #4 // We read 8 extra elements from b - mov x13, x5 - movi v6.8h, #3 - movi v7.4s, #3 -1: - ld1 {v0.8h, v1.8h}, [x9], #32 - ld1 {v2.8h, v3.8h}, [x4], #32 - ld1 {v4.8h, v5.8h}, [x10], #32 - ld1 {v16.4s, v17.4s, v18.4s}, [x7], #48 - ld1 {v19.4s, v20.4s, v21.4s}, [x3], #48 - ld1 {v22.4s, v23.4s, v24.4s}, [x8], #48 - -2: - subs x5, x5, #8 - ext v25.16b, v0.16b, v1.16b, #2 // -stride - ext v26.16b, v2.16b, v3.16b, #2 // 0 - ext v27.16b, v4.16b, v5.16b, #2 // +stride - ext v28.16b, v0.16b, v1.16b, #4 // +1-stride - ext v29.16b, v2.16b, v3.16b, #4 // +1 - ext v30.16b, v4.16b, v5.16b, #4 // +1+stride - add v2.8h, v2.8h, v25.8h // -1, -stride - add v26.8h, v26.8h, v27.8h // 0, +stride - add v0.8h, v0.8h, v28.8h // -1-stride, +1-stride - add v2.8h, v2.8h, v26.8h - add v4.8h, v4.8h, v30.8h // -1+stride, +1+stride - add v2.8h, v2.8h, v29.8h // +1 - add v0.8h, v0.8h, v4.8h - - ext v25.16b, v16.16b, v17.16b, #4 // -stride - ext v26.16b, v17.16b, v18.16b, #4 - shl v2.8h, v2.8h, #2 - ext v27.16b, v16.16b, v17.16b, #8 // +1-stride - ext v28.16b, v17.16b, v18.16b, #8 - ext v29.16b, v19.16b, v20.16b, #4 // 0 - ext v30.16b, v20.16b, v21.16b, #4 - mla v2.8h, v0.8h, v6.8h // * 3 -> a - add v25.4s, v25.4s, v19.4s // -stride, -1 - add v26.4s, v26.4s, v20.4s - add v16.4s, v16.4s, v27.4s // -1-stride, +1-stride - add v17.4s, v17.4s, v28.4s - ext v27.16b, v19.16b, v20.16b, #8 // +1 - ext v28.16b, v20.16b, v21.16b, #8 - add v16.4s, v16.4s, v22.4s // -1+stride - add v17.4s, v17.4s, v23.4s - add v29.4s, v29.4s, v27.4s // 0, +1 - add v30.4s, v30.4s, v28.4s - add v25.4s, v25.4s, v29.4s - add v26.4s, v26.4s, v30.4s - ext v27.16b, v22.16b, v23.16b, #4 // +stride - ext v28.16b, v23.16b, v24.16b, #4 - ext v29.16b, v22.16b, v23.16b, #8 // +1+stride - ext v30.16b, v23.16b, v24.16b, #8 - ld1 {v19.8b}, [x1], #8 // src - add v25.4s, v25.4s, v27.4s // +stride - add v26.4s, v26.4s, v28.4s - add v16.4s, v16.4s, v29.4s // +1+stride - add v17.4s, v17.4s, v30.4s - shl v25.4s, v25.4s, #2 - shl v26.4s, v26.4s, #2 - mla v25.4s, v16.4s, v7.4s // * 3 -> b - mla v26.4s, v17.4s, v7.4s - uxtl v19.8h, v19.8b // src - mov v0.16b, v1.16b - umlal v25.4s, v2.4h, v19.4h // b + a * src - umlal2 v26.4s, v2.8h, v19.8h - mov v2.16b, v3.16b - rshrn v25.4h, v25.4s, #9 - rshrn2 v25.8h, v26.4s, #9 - mov v4.16b, v5.16b - st1 {v25.8h}, [x0], #16 - - b.le 3f - mov v16.16b, v18.16b - mov v19.16b, v21.16b - mov v22.16b, v24.16b - ld1 {v1.8h}, [x9], #16 - ld1 {v3.8h}, [x4], #16 - ld1 {v5.8h}, [x10], #16 - ld1 {v17.4s, v18.4s}, [x7], #32 - ld1 {v20.4s, v21.4s}, [x3], #32 - ld1 {v23.4s, v24.4s}, [x8], #32 - b 2b - -3: - subs x6, x6, #1 - b.le 0f - mov x5, x13 - add x0, x0, x12, lsl #1 - add x1, x1, x2 - add x3, x3, x11, lsl #2 - add x7, x7, x11, lsl #2 - add x8, x8, x11, lsl #2 - add x4, x4, x14, lsl #1 - add x9, x9, x14, lsl #1 - add x10, x10, x14, lsl #1 - b 1b -0: - ret -endfunc - -// void dav1d_sgr_finish_filter2_neon(coef *tmp, -// const pixel *src, const ptrdiff_t stride, -// const int32_t *a, const int16_t *b, -// const int w, const int h); -function sgr_finish_filter2_neon, export=1 - add x7, x3, #(4*(SUM_STRIDE)) - sub x3, x3, #(4*(SUM_STRIDE)) - add x8, x4, #(2*(SUM_STRIDE)) - sub x4, x4, #(2*(SUM_STRIDE)) - mov x9, #(2*SUM_STRIDE) - mov x10, #FILTER_OUT_STRIDE - add x11, x5, #7 - bic x11, x11, #7 // Aligned width - sub x2, x2, x11 - sub x10, x10, x11 - sub x9, x9, x11 - sub x9, x9, #4 // We read 4 extra elements from a - sub x12, x9, #4 // We read 8 extra elements from b - mov x11, x5 - movi v4.8h, #5 - movi v5.4s, #5 - movi v6.8h, #6 - movi v7.4s, #6 -1: - ld1 {v0.8h, v1.8h}, [x4], #32 - ld1 {v2.8h, v3.8h}, [x8], #32 - ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 - ld1 {v19.4s, v20.4s, v21.4s}, [x7], #48 - -2: - subs x5, x5, #8 - ext v24.16b, v0.16b, v1.16b, #4 // +1-stride - ext v25.16b, v2.16b, v3.16b, #4 // +1+stride - ext v22.16b, v0.16b, v1.16b, #2 // -stride - ext v23.16b, v2.16b, v3.16b, #2 // +stride - add v0.8h, v0.8h, v24.8h // -1-stride, +1-stride - add v25.8h, v2.8h, v25.8h // -1+stride, +1+stride - add v2.8h, v22.8h, v23.8h // -stride, +stride - add v0.8h, v0.8h, v25.8h - - ext v22.16b, v16.16b, v17.16b, #4 // -stride - ext v23.16b, v17.16b, v18.16b, #4 - ext v24.16b, v19.16b, v20.16b, #4 // +stride - ext v25.16b, v20.16b, v21.16b, #4 - ext v26.16b, v16.16b, v17.16b, #8 // +1-stride - ext v27.16b, v17.16b, v18.16b, #8 - ext v28.16b, v19.16b, v20.16b, #8 // +1+stride - ext v29.16b, v20.16b, v21.16b, #8 - mul v0.8h, v0.8h, v4.8h // * 5 - mla v0.8h, v2.8h, v6.8h // * 6 - ld1 {v31.8b}, [x1], #8 - add v16.4s, v16.4s, v26.4s // -1-stride, +1-stride - add v17.4s, v17.4s, v27.4s - add v19.4s, v19.4s, v28.4s // -1+stride, +1+stride - add v20.4s, v20.4s, v29.4s - add v16.4s, v16.4s, v19.4s - add v17.4s, v17.4s, v20.4s - - add v22.4s, v22.4s, v24.4s // -stride, +stride - add v23.4s, v23.4s, v25.4s - // This is, surprisingly, faster than other variants where the - // mul+mla pairs are further apart, on Cortex A53. - mul v16.4s, v16.4s, v5.4s // * 5 - mla v16.4s, v22.4s, v7.4s // * 6 - mul v17.4s, v17.4s, v5.4s // * 5 - mla v17.4s, v23.4s, v7.4s // * 6 - - uxtl v31.8h, v31.8b - umlal v16.4s, v0.4h, v31.4h // b + a * src - umlal2 v17.4s, v0.8h, v31.8h - mov v0.16b, v1.16b - rshrn v16.4h, v16.4s, #9 - rshrn2 v16.8h, v17.4s, #9 - mov v2.16b, v3.16b - st1 {v16.8h}, [x0], #16 - - b.le 3f - mov v16.16b, v18.16b - mov v19.16b, v21.16b - ld1 {v1.8h}, [x4], #16 - ld1 {v3.8h}, [x8], #16 - ld1 {v17.4s, v18.4s}, [x3], #32 - ld1 {v20.4s, v21.4s}, [x7], #32 - b 2b - -3: - subs x6, x6, #1 - b.le 0f - mov x5, x11 - add x0, x0, x10, lsl #1 - add x1, x1, x2 - add x3, x3, x9, lsl #2 - add x7, x7, x9, lsl #2 - add x4, x4, x12, lsl #1 - add x8, x8, x12, lsl #1 - mov x13, x3 - mov x14, x4 - - ld1 {v0.8h, v1.8h}, [x4], #32 - ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 - -4: - subs x5, x5, #8 - ext v23.16b, v0.16b, v1.16b, #4 // +1 - ext v22.16b, v0.16b, v1.16b, #2 // 0 - add v0.8h, v0.8h, v23.8h // -1, +1 - - ext v24.16b, v16.16b, v17.16b, #4 // 0 - ext v25.16b, v17.16b, v18.16b, #4 - ext v26.16b, v16.16b, v17.16b, #8 // +1 - ext v27.16b, v17.16b, v18.16b, #8 - mul v2.8h, v22.8h, v6.8h // * 6 - mla v2.8h, v0.8h, v4.8h // * 5 -> a - ld1 {v31.8b}, [x1], #8 - add v16.4s, v16.4s, v26.4s // -1, +1 - add v17.4s, v17.4s, v27.4s - uxtl v31.8h, v31.8b - // This is, surprisingly, faster than other variants where the - // mul+mla pairs are further apart, on Cortex A53. - mul v24.4s, v24.4s, v7.4s // * 6 - mla v24.4s, v16.4s, v5.4s // * 5 -> b - mul v25.4s, v25.4s, v7.4s // * 6 - mla v25.4s, v17.4s, v5.4s // * 5 -> b - - umlal v24.4s, v2.4h, v31.4h // b + a * src - umlal2 v25.4s, v2.8h, v31.8h - mov v0.16b, v1.16b - rshrn v24.4h, v24.4s, #8 - rshrn2 v24.8h, v25.4s, #8 - mov v16.16b, v18.16b - st1 {v24.8h}, [x0], #16 - - b.le 5f - ld1 {v1.8h}, [x4], #16 - ld1 {v17.4s, v18.4s}, [x3], #32 - b 4b - -5: - subs x6, x6, #1 - b.le 0f - mov x5, x11 - add x0, x0, x10, lsl #1 - add x1, x1, x2 - mov x3, x13 // Rewind x3/x4 to where they started - mov x4, x14 - b 1b -0: - ret -endfunc - -// void dav1d_sgr_weighted1_neon(pixel *dst, const ptrdiff_t dst_stride, -// const pixel *src, const ptrdiff_t src_stride, -// const coef *t1, const int w, const int h, -// const int wt); -function sgr_weighted1_neon, export=1 - dup v31.8h, w7 - cmp x6, #2 - add x9, x0, x1 - add x10, x2, x3 - add x11, x4, #2*FILTER_OUT_STRIDE - mov x7, #(4*FILTER_OUT_STRIDE) - lsl x1, x1, #1 - lsl x3, x3, #1 - add x8, x5, #7 - bic x8, x8, #7 // Aligned width - sub x1, x1, x8 - sub x3, x3, x8 - sub x7, x7, x8, lsl #1 - mov x8, x5 - b.lt 2f -1: - ld1 {v0.8b}, [x2], #8 - ld1 {v4.8b}, [x10], #8 - ld1 {v1.8h}, [x4], #16 - ld1 {v5.8h}, [x11], #16 - subs x5, x5, #8 - ushll v0.8h, v0.8b, #4 // u - ushll v4.8h, v4.8b, #4 // u - sub v1.8h, v1.8h, v0.8h // t1 - u - sub v5.8h, v5.8h, v4.8h // t1 - u - ushll v2.4s, v0.4h, #7 // u << 7 - ushll2 v3.4s, v0.8h, #7 // u << 7 - ushll v6.4s, v4.4h, #7 // u << 7 - ushll2 v7.4s, v4.8h, #7 // u << 7 - smlal v2.4s, v1.4h, v31.4h // v - smlal2 v3.4s, v1.8h, v31.8h // v - smlal v6.4s, v5.4h, v31.4h // v - smlal2 v7.4s, v5.8h, v31.8h // v - rshrn v2.4h, v2.4s, #11 - rshrn2 v2.8h, v3.4s, #11 - rshrn v6.4h, v6.4s, #11 - rshrn2 v6.8h, v7.4s, #11 - sqxtun v2.8b, v2.8h - sqxtun v6.8b, v6.8h - st1 {v2.8b}, [x0], #8 - st1 {v6.8b}, [x9], #8 - b.gt 1b - - sub x6, x6, #2 - cmp x6, #1 - b.lt 0f - mov x5, x8 - add x0, x0, x1 - add x9, x9, x1 - add x2, x2, x3 - add x10, x10, x3 - add x4, x4, x7 - add x11, x11, x7 - b.eq 2f - b 1b - -2: - ld1 {v0.8b}, [x2], #8 - ld1 {v1.8h}, [x4], #16 - subs x5, x5, #8 - ushll v0.8h, v0.8b, #4 // u - sub v1.8h, v1.8h, v0.8h // t1 - u - ushll v2.4s, v0.4h, #7 // u << 7 - ushll2 v3.4s, v0.8h, #7 // u << 7 - smlal v2.4s, v1.4h, v31.4h // v - smlal2 v3.4s, v1.8h, v31.8h // v - rshrn v2.4h, v2.4s, #11 - rshrn2 v2.8h, v3.4s, #11 - sqxtun v2.8b, v2.8h - st1 {v2.8b}, [x0], #8 - b.gt 2b -0: - ret -endfunc - -// void dav1d_sgr_weighted2_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *src, const ptrdiff_t src_stride, -// const coef *t1, const coef *t2, -// const int w, const int h, -// const int16_t wt[2]); -function sgr_weighted2_neon, export=1 - ldr x8, [sp] - cmp x7, #2 - add x10, x0, x1 - add x11, x2, x3 - add x12, x4, #2*FILTER_OUT_STRIDE - add x13, x5, #2*FILTER_OUT_STRIDE - ld2r {v30.8h, v31.8h}, [x8] // wt[0], wt[1] - mov x8, #4*FILTER_OUT_STRIDE - lsl x1, x1, #1 - lsl x3, x3, #1 - add x9, x6, #7 - bic x9, x9, #7 // Aligned width - sub x1, x1, x9 - sub x3, x3, x9 - sub x8, x8, x9, lsl #1 - mov x9, x6 - b.lt 2f -1: - ld1 {v0.8b}, [x2], #8 - ld1 {v16.8b}, [x11], #8 - ld1 {v1.8h}, [x4], #16 - ld1 {v17.8h}, [x12], #16 - ld1 {v2.8h}, [x5], #16 - ld1 {v18.8h}, [x13], #16 - subs x6, x6, #8 - ushll v0.8h, v0.8b, #4 // u - ushll v16.8h, v16.8b, #4 // u - sub v1.8h, v1.8h, v0.8h // t1 - u - sub v2.8h, v2.8h, v0.8h // t2 - u - sub v17.8h, v17.8h, v16.8h // t1 - u - sub v18.8h, v18.8h, v16.8h // t2 - u - ushll v3.4s, v0.4h, #7 // u << 7 - ushll2 v4.4s, v0.8h, #7 // u << 7 - ushll v19.4s, v16.4h, #7 // u << 7 - ushll2 v20.4s, v16.8h, #7 // u << 7 - smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) - smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) - smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) - smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) - smlal v19.4s, v17.4h, v30.4h // wt[0] * (t1 - u) - smlal v19.4s, v18.4h, v31.4h // wt[1] * (t2 - u) - smlal2 v20.4s, v17.8h, v30.8h // wt[0] * (t1 - u) - smlal2 v20.4s, v18.8h, v31.8h // wt[1] * (t2 - u) - rshrn v3.4h, v3.4s, #11 - rshrn2 v3.8h, v4.4s, #11 - rshrn v19.4h, v19.4s, #11 - rshrn2 v19.8h, v20.4s, #11 - sqxtun v3.8b, v3.8h - sqxtun v19.8b, v19.8h - st1 {v3.8b}, [x0], #8 - st1 {v19.8b}, [x10], #8 - b.gt 1b - - subs x7, x7, #2 - cmp x7, #1 - b.lt 0f - mov x6, x9 - add x0, x0, x1 - add x10, x10, x1 - add x2, x2, x3 - add x11, x11, x3 - add x4, x4, x8 - add x12, x12, x8 - add x5, x5, x8 - add x13, x13, x8 - b.eq 2f - b 1b - -2: - ld1 {v0.8b}, [x2], #8 - ld1 {v1.8h}, [x4], #16 - ld1 {v2.8h}, [x5], #16 - subs x6, x6, #8 - ushll v0.8h, v0.8b, #4 // u - sub v1.8h, v1.8h, v0.8h // t1 - u - sub v2.8h, v2.8h, v0.8h // t2 - u - ushll v3.4s, v0.4h, #7 // u << 7 - ushll2 v4.4s, v0.8h, #7 // u << 7 - smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) - smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) - smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) - smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) - rshrn v3.4h, v3.4s, #11 - rshrn2 v3.8h, v4.4s, #11 - sqxtun v3.8b, v3.8h - st1 {v3.8b}, [x0], #8 - b.gt 1b -0: - ret -endfunc +sgr_funcs 8 diff --git a/third_party/dav1d/src/arm/64/looprestoration16.S b/third_party/dav1d/src/arm/64/looprestoration16.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/looprestoration16.S @@ -0,0 +1,1239 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void dav1d_wiener_filter_h_16bpc_neon(int16_t *dst, const pixel (*left)[4], +// const pixel *src, ptrdiff_t stride, +// const int16_t fh[7], const intptr_t w, +// int h, enum LrEdgeFlags edges, +// const int bitdepth_max); +function wiener_filter_h_16bpc_neon, export=1 + ldr w8, [sp] // bitdepth_max + ld1 {v0.8h}, [x4] + clz w8, w8 + movi v30.4s, #1 + sub w9, w8, #38 // -(bitdepth + 6) + sub w8, w8, #25 // -round_bits_h + neg w9, w9 // bitdepth + 6 + dup v1.4s, w9 + dup v29.4s, w8 // -round_bits_h + movi v31.8h, #0x20, lsl #8 // 1 << 13 = 8192 + ushl v30.4s, v30.4s, v1.4s // 1 << (bitdepth + 6) + mov w8, w5 + // Calculate mid_stride + add w10, w5, #7 + bic w10, w10, #7 + lsl w10, w10, #1 + + // Clear the last unused element of v0, to allow filtering a single + // pixel with one plain mul+addv. + ins v0.h[7], wzr + + // Set up pointers for reading/writing alternate rows + add x12, x0, x10 + lsl w10, w10, #1 + add x13, x2, x3 + lsl x3, x3, #1 + + // Subtract the width from mid_stride + sub x10, x10, w5, uxtw #1 + + // For w >= 8, we read (w+5)&~7+8 pixels, for w < 8 we read 16 pixels. + cmp w5, #8 + add w11, w5, #13 + bic w11, w11, #7 + b.ge 1f + mov w11, #16 +1: + sub x3, x3, w11, uxtw #1 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 2f + // LR_HAVE_LEFT + cbnz x1, 0f + // left == NULL + sub x2, x2, #6 + sub x13, x13, #6 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add x3, x3, #6 + + +1: // Loop vertically + ld1 {v2.8h, v3.8h}, [x2], #32 + ld1 {v4.8h, v5.8h}, [x13], #32 + + tst w7, #1 // LR_HAVE_LEFT + b.eq 0f + cbz x1, 2f + // LR_HAVE_LEFT, left != NULL + ld1 {v1.d}[1], [x1], #8 + // Move x2/x13 back to account for the last 3 pixels we loaded earlier, + // which we'll shift out. + sub x2, x2, #6 + sub x13, x13, #6 + ld1 {v6.d}[1], [x1], #8 + ext v3.16b, v2.16b, v3.16b, #10 + ext v2.16b, v1.16b, v2.16b, #10 + ext v5.16b, v4.16b, v5.16b, #10 + ext v4.16b, v6.16b, v4.16b, #10 + b 2f +0: + // !LR_HAVE_LEFT, fill v1 with the leftmost pixel + // and shift v2/v3 to have 3x the first pixel at the front. + dup v1.8h, v2.h[0] + dup v6.8h, v4.h[0] + // Move x2 back to account for the last 3 pixels we loaded before, + // which we shifted out. + sub x2, x2, #6 + sub x13, x13, #6 + ext v3.16b, v2.16b, v3.16b, #10 + ext v2.16b, v1.16b, v2.16b, #10 + ext v5.16b, v4.16b, v5.16b, #10 + ext v4.16b, v6.16b, v4.16b, #10 + +2: + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub w9, w5, #14 + ldr h27, [x2, w9, sxtw #1] + ldr h28, [x13, w9, sxtw #1] + // Fill v27/v28 with the right padding pixel + dup v27.8h, v27.h[0] + dup v28.8h, v28.h[0] +3: // !LR_HAVE_RIGHT + // If we'll have to pad the right edge we need to quit early here. + cmp w5, #11 + b.ge 4f // If w >= 11, all used input pixels are valid + cmp w5, #7 + b.ge 5f // If w >= 7, we can filter 4 pixels + b 6f + +4: // Loop horizontally +.macro ushll_sz d0, d1, src, shift, wd + ushll \d0\().4s, \src\().4h, \shift +.ifc \wd, .8h + ushll2 \d1\().4s, \src\().8h, \shift +.endif +.endm +.macro add_sz d0, d1, s0, s1, c, wd + add \d0\().4s, \s0\().4s, \c\().4s +.ifc \wd, .8h + add \d1\().4s, \s1\().4s, \c\().4s +.endif +.endm +.macro srshl_sz d0, d1, s0, s1, c, wd + srshl \d0\().4s, \s0\().4s, \c\().4s +.ifc \wd, .8h + srshl \d1\().4s, \s1\().4s, \c\().4s +.endif +.endm +.macro sqxtun_sz dst, s0, s1, wd + sqxtun \dst\().4h, \s0\().4s +.ifc \wd, .8h + sqxtun2 \dst\().8h, \s1\().4s +.endif +.endm + +.macro filter wd + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + ext v16.16b, v2.16b, v3.16b, #2 + ext v17.16b, v2.16b, v3.16b, #4 + ext v18.16b, v2.16b, v3.16b, #6 + ext v19.16b, v2.16b, v3.16b, #8 + ext v20.16b, v2.16b, v3.16b, #10 + ext v21.16b, v2.16b, v3.16b, #12 + ushll_sz v6, v7, v18, #7, \wd + smlal v6.4s, v2.4h, v0.h[0] + smlal v6.4s, v16.4h, v0.h[1] + smlal v6.4s, v17.4h, v0.h[2] + smlal v6.4s, v18.4h, v0.h[3] + smlal v6.4s, v19.4h, v0.h[4] + smlal v6.4s, v20.4h, v0.h[5] + smlal v6.4s, v21.4h, v0.h[6] +.ifc \wd, .8h + smlal2 v7.4s, v2.8h, v0.h[0] + smlal2 v7.4s, v16.8h, v0.h[1] + smlal2 v7.4s, v17.8h, v0.h[2] + smlal2 v7.4s, v18.8h, v0.h[3] + smlal2 v7.4s, v19.8h, v0.h[4] + smlal2 v7.4s, v20.8h, v0.h[5] + smlal2 v7.4s, v21.8h, v0.h[6] +.endif + ext v19.16b, v4.16b, v5.16b, #2 + ext v20.16b, v4.16b, v5.16b, #4 + ext v21.16b, v4.16b, v5.16b, #6 + ext v22.16b, v4.16b, v5.16b, #8 + ext v23.16b, v4.16b, v5.16b, #10 + ext v24.16b, v4.16b, v5.16b, #12 + ushll_sz v16, v17, v21, #7, \wd + smlal v16.4s, v4.4h, v0.h[0] + smlal v16.4s, v19.4h, v0.h[1] + smlal v16.4s, v20.4h, v0.h[2] + smlal v16.4s, v21.4h, v0.h[3] + smlal v16.4s, v22.4h, v0.h[4] + smlal v16.4s, v23.4h, v0.h[5] + smlal v16.4s, v24.4h, v0.h[6] +.ifc \wd, .8h + smlal2 v17.4s, v4.8h, v0.h[0] + smlal2 v17.4s, v19.8h, v0.h[1] + smlal2 v17.4s, v20.8h, v0.h[2] + smlal2 v17.4s, v21.8h, v0.h[3] + smlal2 v17.4s, v22.8h, v0.h[4] + smlal2 v17.4s, v23.8h, v0.h[5] + smlal2 v17.4s, v24.8h, v0.h[6] +.endif + mvni v24\wd, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1 + add_sz v6, v7, v6, v7, v30, \wd + add_sz v16, v17, v16, v17, v30, \wd + srshl_sz v6, v7, v6, v7, v29, \wd + srshl_sz v16, v17, v16, v17, v29, \wd + sqxtun_sz v6, v6, v7, \wd + sqxtun_sz v7, v16, v17, \wd + umin v6\wd, v6\wd, v24\wd + umin v7\wd, v7\wd, v24\wd + sub v6\wd, v6\wd, v31\wd + sub v7\wd, v7\wd, v31\wd +.endm + filter .8h + st1 {v6.8h}, [x0], #16 + st1 {v7.8h}, [x12], #16 + + subs w5, w5, #8 + b.le 9f + tst w7, #2 // LR_HAVE_RIGHT + mov v2.16b, v3.16b + mov v4.16b, v5.16b + ld1 {v3.8h}, [x2], #16 + ld1 {v5.8h}, [x13], #16 + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +5: // Filter 4 pixels, 7 <= w < 11 + filter .4h + st1 {v6.4h}, [x0], #8 + st1 {v7.4h}, [x12], #8 + + subs w5, w5, #4 // 3 <= w < 7 + ext v2.16b, v2.16b, v3.16b, #8 + ext v3.16b, v3.16b, v3.16b, #8 + ext v4.16b, v4.16b, v5.16b, #8 + ext v5.16b, v5.16b, v5.16b, #8 + +6: // Pad the right edge and filter the last few pixels. + // w < 7, w+3 pixels valid in v2-v3 + cmp w5, #5 + b.lt 7f + b.gt 8f + // w == 5, 8 pixels valid in v2, v3 invalid + mov v3.16b, v27.16b + mov v5.16b, v28.16b + b 88f + +7: // 1 <= w < 5, 4-7 pixels valid in v2 + sub w9, w5, #1 + // w9 = (pixels valid - 4) + adr x11, L(variable_shift_tbl) + ldrh w9, [x11, w9, uxtw #1] + sub x11, x11, w9, uxth + mov v3.16b, v27.16b + mov v5.16b, v28.16b + br x11 +44: // 4 pixels valid in v2/v4, fill the high half with padding. + ins v2.d[1], v3.d[0] + ins v4.d[1], v5.d[0] + b 88f + // Shift v2 right, shifting out invalid pixels, + // shift v2 left to the original offset, shifting in padding pixels. +55: // 5 pixels valid + ext v2.16b, v2.16b, v2.16b, #10 + ext v2.16b, v2.16b, v3.16b, #6 + ext v4.16b, v4.16b, v4.16b, #10 + ext v4.16b, v4.16b, v5.16b, #6 + b 88f +66: // 6 pixels valid, fill the upper 2 pixels with padding. + ins v2.s[3], v3.s[0] + ins v4.s[3], v5.s[0] + b 88f +77: // 7 pixels valid, fill the last pixel with padding. + ins v2.h[7], v3.h[0] + ins v4.h[7], v5.h[0] + b 88f + +L(variable_shift_tbl): + .hword L(variable_shift_tbl) - 44b + .hword L(variable_shift_tbl) - 55b + .hword L(variable_shift_tbl) - 66b + .hword L(variable_shift_tbl) - 77b + +8: // w > 5, w == 6, 9 pixels valid in v2-v3, 1 pixel valid in v3 + ins v27.h[0], v3.h[0] + ins v28.h[0], v5.h[0] + mov v3.16b, v27.16b + mov v5.16b, v28.16b + +88: + // w < 7, v2-v3 padded properly + cmp w5, #4 + b.lt 888f + + // w >= 4, filter 4 pixels + filter .4h + st1 {v6.4h}, [x0], #8 + st1 {v7.4h}, [x12], #8 + subs w5, w5, #4 // 0 <= w < 4 + ext v2.16b, v2.16b, v3.16b, #8 + ext v4.16b, v4.16b, v5.16b, #8 + b.eq 9f +888: // 1 <= w < 4, filter 1 pixel at a time + smull v6.4s, v2.4h, v0.4h + smull2 v7.4s, v2.8h, v0.8h + smull v16.4s, v4.4h, v0.4h + smull2 v17.4s, v4.8h, v0.8h + add v6.4s, v6.4s, v7.4s + add v16.4s, v16.4s, v17.4s + addv s6, v6.4s + addv s7, v16.4s + dup v16.4h, v2.h[3] + ins v16.h[1], v4.h[3] + ins v6.s[1], v7.s[0] + mvni v24.4h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1 + ushll v16.4s, v16.4h, #7 + add v6.4s, v6.4s, v30.4s + add v6.4s, v6.4s, v16.4s + srshl v6.4s, v6.4s, v29.4s + sqxtun v6.4h, v6.4s + umin v6.4h, v6.4h, v24.4h + sub v6.4h, v6.4h, v31.4h + st1 {v6.h}[0], [x0], #2 + st1 {v6.h}[1], [x12], #2 + subs w5, w5, #1 + ext v2.16b, v2.16b, v3.16b, #2 + ext v4.16b, v4.16b, v5.16b, #2 + b.gt 888b + +9: + subs w6, w6, #2 + b.le 0f + // Jump to the next row and loop horizontally + add x0, x0, x10 + add x12, x12, x10 + add x2, x2, x3 + add x13, x13, x3 + mov w5, w8 + b 1b +0: + ret +.purgem filter +endfunc + +// void dav1d_wiener_filter_v_16bpc_neon(pixel *dst, ptrdiff_t stride, +// const int16_t *mid, int w, int h, +// const int16_t fv[7], enum LrEdgeFlags edges, +// ptrdiff_t mid_stride, const int bitdepth_max); +function wiener_filter_v_16bpc_neon, export=1 + ldr w8, [sp] // bitdepth_max + ld1 {v0.8h}, [x5] + dup v31.8h, w8 + clz w8, w8 + movi v1.8h, #128 + sub w8, w8, #11 // round_bits_v + add v1.8h, v1.8h, v0.8h + dup v30.4s, w8 + mov w8, w4 + neg v30.4s, v30.4s // -round_bits_v + + // Calculate the number of rows to move back when looping vertically + mov w11, w4 + tst w6, #4 // LR_HAVE_TOP + b.eq 0f + sub x2, x2, x7, lsl #1 + add w11, w11, #2 +0: + tst w6, #8 // LR_HAVE_BOTTOM + b.eq 1f + add w11, w11, #2 + +1: // Start of horizontal loop; start one vertical filter slice. + // Load rows into v16-v19 and pad properly. + tst w6, #4 // LR_HAVE_TOP + ld1 {v16.8h}, [x2], x7 + b.eq 2f + // LR_HAVE_TOP + ld1 {v18.8h}, [x2], x7 + mov v17.16b, v16.16b + ld1 {v19.8h}, [x2], x7 + b 3f +2: // !LR_HAVE_TOP + mov v17.16b, v16.16b + mov v18.16b, v16.16b + mov v19.16b, v16.16b + +3: + cmp w4, #4 + b.lt 5f + // Start filtering normally; fill in v20-v22 with unique rows. + ld1 {v20.8h}, [x2], x7 + ld1 {v21.8h}, [x2], x7 + ld1 {v22.8h}, [x2], x7 + +4: +.macro filter compare + subs w4, w4, #1 + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + smull v2.4s, v16.4h, v0.h[0] + smlal v2.4s, v17.4h, v0.h[1] + smlal v2.4s, v18.4h, v0.h[2] + smlal v2.4s, v19.4h, v1.h[3] + smlal v2.4s, v20.4h, v0.h[4] + smlal v2.4s, v21.4h, v0.h[5] + smlal v2.4s, v22.4h, v0.h[6] + smull2 v3.4s, v16.8h, v0.h[0] + smlal2 v3.4s, v17.8h, v0.h[1] + smlal2 v3.4s, v18.8h, v0.h[2] + smlal2 v3.4s, v19.8h, v1.h[3] + smlal2 v3.4s, v20.8h, v0.h[4] + smlal2 v3.4s, v21.8h, v0.h[5] + smlal2 v3.4s, v22.8h, v0.h[6] + srshl v2.4s, v2.4s, v30.4s // round_bits_v + srshl v3.4s, v3.4s, v30.4s + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + umin v2.8h, v2.8h, v31.8h // bitdepth_max + st1 {v2.8h}, [x0], x1 +.if \compare + cmp w4, #4 +.else + b.le 9f +.endif + mov v16.16b, v17.16b + mov v17.16b, v18.16b + mov v18.16b, v19.16b + mov v19.16b, v20.16b + mov v20.16b, v21.16b + mov v21.16b, v22.16b +.endm + filter 1 + b.lt 7f + ld1 {v22.8h}, [x2], x7 + b 4b + +5: // Less than 4 rows in total; not all of v20-v21 are filled yet. + tst w6, #8 // LR_HAVE_BOTTOM + b.eq 6f + // LR_HAVE_BOTTOM + cmp w4, #2 + // We load at least 2 rows in all cases. + ld1 {v20.8h}, [x2], x7 + ld1 {v21.8h}, [x2], x7 + b.gt 53f // 3 rows in total + b.eq 52f // 2 rows in total +51: // 1 row in total, v19 already loaded, load edge into v20-v22. + mov v22.16b, v21.16b + b 8f +52: // 2 rows in total, v19 already loaded, load v20 with content data + // and 2 rows of edge. + ld1 {v22.8h}, [x2], x7 + mov v23.16b, v22.16b + b 8f +53: + // 3 rows in total, v19 already loaded, load v20 and v21 with content + // and 2 rows of edge. + ld1 {v22.8h}, [x2], x7 + ld1 {v23.8h}, [x2], x7 + mov v24.16b, v23.16b + b 8f + +6: + // !LR_HAVE_BOTTOM + cmp w4, #2 + b.gt 63f // 3 rows in total + b.eq 62f // 2 rows in total +61: // 1 row in total, v19 already loaded, pad that into v20-v22. + mov v20.16b, v19.16b + mov v21.16b, v19.16b + mov v22.16b, v19.16b + b 8f +62: // 2 rows in total, v19 already loaded, load v20 and pad that into v21-v23. + ld1 {v20.8h}, [x2], x7 + mov v21.16b, v20.16b + mov v22.16b, v20.16b + mov v23.16b, v20.16b + b 8f +63: + // 3 rows in total, v19 already loaded, load v20 and v21 and pad v21 into v22-v24. + ld1 {v20.8h}, [x2], x7 + ld1 {v21.8h}, [x2], x7 + mov v22.16b, v21.16b + mov v23.16b, v21.16b + mov v24.16b, v21.16b + b 8f + +7: + // All registers up to v21 are filled already, 3 valid rows left. + // < 4 valid rows left; fill in padding and filter the last + // few rows. + tst w6, #8 // LR_HAVE_BOTTOM + b.eq 71f + // LR_HAVE_BOTTOM; load 2 rows of edge. + ld1 {v22.8h}, [x2], x7 + ld1 {v23.8h}, [x2], x7 + mov v24.16b, v23.16b + b 8f +71: + // !LR_HAVE_BOTTOM, pad 3 rows + mov v22.16b, v21.16b + mov v23.16b, v21.16b + mov v24.16b, v21.16b + +8: // At this point, all registers up to v22-v24 are loaded with + // edge/padding (depending on how many rows are left). + filter 0 // This branches to 9f when done + mov v22.16b, v23.16b + mov v23.16b, v24.16b + b 8b + +9: // End of one vertical slice. + subs w3, w3, #8 + b.le 0f + // Move pointers back up to the top and loop horizontally. + msub x0, x1, x8, x0 + msub x2, x7, x11, x2 + add x0, x0, #16 + add x2, x2, #16 + mov w4, w8 + b 1b + +0: + ret +.purgem filter +endfunc + +// void dav1d_copy_narrow_16bpc_neon(pixel *dst, ptrdiff_t stride, +// const pixel *src, int w, int h); +function copy_narrow_16bpc_neon, export=1 + adr x5, L(copy_narrow_tbl) + ldrh w6, [x5, w3, uxtw #1] + sub x5, x5, w6, uxth + br x5 +10: + add x7, x0, x1 + lsl x1, x1, #1 +18: + subs w4, w4, #8 + b.lt 110f + ld1 {v0.8h}, [x2], #16 + st1 {v0.h}[0], [x0], x1 + st1 {v0.h}[1], [x7], x1 + st1 {v0.h}[2], [x0], x1 + st1 {v0.h}[3], [x7], x1 + st1 {v0.h}[4], [x0], x1 + st1 {v0.h}[5], [x7], x1 + st1 {v0.h}[6], [x0], x1 + st1 {v0.h}[7], [x7], x1 + b.le 0f + b 18b +110: + add w4, w4, #8 + asr x1, x1, #1 +11: + subs w4, w4, #1 + ld1 {v0.h}[0], [x2], #2 + st1 {v0.h}[0], [x0], x1 + b.gt 11b +0: + ret + +20: + add x7, x0, x1 + lsl x1, x1, #1 +24: + subs w4, w4, #4 + b.lt 210f + ld1 {v0.4s}, [x2], #16 + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[1], [x7], x1 + st1 {v0.s}[2], [x0], x1 + st1 {v0.s}[3], [x7], x1 + b.le 0f + b 24b +210: + add w4, w4, #4 + asr x1, x1, #1 +22: + subs w4, w4, #1 + ld1 {v0.s}[0], [x2], #4 + st1 {v0.s}[0], [x0], x1 + b.gt 22b +0: + ret + +30: + ldr w5, [x2] + ldrh w6, [x2, #4] + add x2, x2, #6 + subs w4, w4, #1 + str w5, [x0] + strh w6, [x0, #4] + add x0, x0, x1 + b.gt 30b + ret + +40: + add x7, x0, x1 + lsl x1, x1, #1 +42: + subs w4, w4, #2 + b.lt 41f + ld1 {v0.2d}, [x2], #16 + st1 {v0.d}[0], [x0], x1 + st1 {v0.d}[1], [x7], x1 + b.le 0f + b 42b +41: + ld1 {v0.4h}, [x2] + st1 {v0.4h}, [x0] +0: + ret + +50: + ldr x5, [x2] + ldrh w6, [x2, #8] + add x2, x2, #10 + subs w4, w4, #1 + str x5, [x0] + strh w6, [x0, #8] + add x0, x0, x1 + b.gt 50b + ret + +60: + ldr x5, [x2] + ldr w6, [x2, #8] + add x2, x2, #12 + subs w4, w4, #1 + str x5, [x0] + str w6, [x0, #8] + add x0, x0, x1 + b.gt 60b + ret + +70: + ldr x5, [x2] + ldr w6, [x2, #8] + ldrh w7, [x2, #12] + add x2, x2, #14 + subs w4, w4, #1 + str x5, [x0] + str w6, [x0, #8] + strh w7, [x0, #12] + add x0, x0, x1 + b.gt 70b + ret + +L(copy_narrow_tbl): + .hword 0 + .hword L(copy_narrow_tbl) - 10b + .hword L(copy_narrow_tbl) - 20b + .hword L(copy_narrow_tbl) - 30b + .hword L(copy_narrow_tbl) - 40b + .hword L(copy_narrow_tbl) - 50b + .hword L(copy_narrow_tbl) - 60b + .hword L(copy_narrow_tbl) - 70b +endfunc + +#define SUM_STRIDE (384+16) + +#include "looprestoration_tmpl.S" + +// void dav1d_sgr_box3_h_16bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_h_16bpc_neon, export=1 + add w5, w5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add x10, x0, #(4*SUM_STRIDE) // sumsq + add x11, x1, #(2*SUM_STRIDE) // sum + add x12, x3, x4 // src + lsl x4, x4, #1 + mov x9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + // With LR_HAVE_RIGHT, align to 8, without it, align to 4. + tst w7, #2 // LR_HAVE_RIGHT + b.ne 0f + // !LR_HAVE_RIGHT + add w13, w5, #3 + bic w13, w13, #3 + b 1f +0: + add w13, w5, #7 + bic w13, w13, #7 +1: + sub x9, x9, w13, uxtw #1 + + // Store the width for the vertical loop + mov w8, w5 + + // Subtract the number of pixels read from the input from the stride + add w13, w5, #14 + bic w13, w13, #7 + sub x4, x4, w13, uxtw #1 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 2f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #4 + sub x12, x12, #4 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 2 pixels from the src pointer, + // but shift it as if we had done that. + add x4, x4, #4 + + +1: // Loop vertically + ld1 {v0.8h, v1.8h}, [x3], #32 + ld1 {v16.8h, v17.8h}, [x12], #32 + + tst w7, #1 // LR_HAVE_LEFT + b.eq 0f + cbz x2, 2f + // LR_HAVE_LEFT, left != NULL + ld1 {v2.d}[1], [x2], #8 + // Move x3/x12 back to account for the last 2 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #4 + sub x12, x12, #4 + ld1 {v18.d}[1], [x2], #8 + ext v1.16b, v0.16b, v1.16b, #12 + ext v0.16b, v2.16b, v0.16b, #12 + ext v17.16b, v16.16b, v17.16b, #12 + ext v16.16b, v18.16b, v16.16b, #12 + b 2f +0: + // !LR_HAVE_LEFT, fill v2 with the leftmost pixel + // and shift v0/v1 to have 2x the first pixel at the front. + dup v2.8h, v0.h[0] + dup v18.8h, v16.h[0] + // Move x3 back to account for the last 2 pixels we loaded before, + // which we shifted out. + sub x3, x3, #4 + sub x12, x12, #4 + ext v1.16b, v0.16b, v1.16b, #12 + ext v0.16b, v2.16b, v0.16b, #12 + ext v17.16b, v16.16b, v17.16b, #12 + ext v16.16b, v18.16b, v16.16b, #12 + +2: + umull v2.4s, v0.4h, v0.4h + umull2 v3.4s, v0.8h, v0.8h + umull v4.4s, v1.4h, v1.4h + umull v18.4s, v16.4h, v16.4h + umull2 v19.4s, v16.8h, v16.8h + umull v20.4s, v17.4h, v17.4h + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub w13, w5, #(2 + 16 - 2 + 1) + ldr h30, [x3, w13, sxtw #1] + ldr h31, [x12, w13, sxtw #1] + // Fill v30/v31 with the right padding pixel + dup v30.8h, v30.h[0] + dup v31.8h, v31.h[0] +3: // !LR_HAVE_RIGHT + // If we'll have to pad the right edge we need to quit early here. + cmp w5, #10 + b.ge 4f // If w >= 10, all used input pixels are valid + cmp w5, #6 + b.ge 5f // If w >= 6, we can filter 4 pixels + b 6f + +4: // Loop horizontally +.macro ext_n dst1, dst2, src1, src2, src3, n, w + ext \dst1, \src1, \src2, \n +.if \w > 4 + ext \dst2, \src2, \src3, \n +.endif +.endm +.macro add_n dst1, dst2, src1, src2, src3, src4, w + add \dst1, \src1, \src3 +.if \w > 4 + add \dst2, \src2, \src4 +.endif +.endm + +.macro add3 w, wd + ext v24.16b, v0.16b, v1.16b, #2 + ext v25.16b, v0.16b, v1.16b, #4 + ext v26.16b, v16.16b, v17.16b, #2 + ext v27.16b, v16.16b, v17.16b, #4 + add v6\wd, v0\wd, v24\wd + add v7\wd, v16\wd, v26\wd + add v6\wd, v6\wd, v25\wd + add v7\wd, v7\wd, v27\wd + + ext_n v24.16b, v25.16b, v2.16b, v3.16b, v4.16b, #4, \w + ext_n v26.16b, v27.16b, v2.16b, v3.16b, v4.16b, #8, \w + + add_n v22.4s, v23.4s, v2.4s, v3.4s, v24.4s, v25.4s, \w + add_n v22.4s, v23.4s, v22.4s, v23.4s, v26.4s, v27.4s, \w + + ext_n v24.16b, v25.16b, v18.16b, v19.16b, v20.16b, #4, \w + ext_n v26.16b, v27.16b, v18.16b, v19.16b, v20.16b, #8, \w + + add_n v24.4s, v25.4s, v18.4s, v19.4s, v24.4s, v25.4s, \w + add_n v24.4s, v25.4s, v24.4s, v25.4s, v26.4s, v27.4s, \w +.endm + add3 8, .8h + st1 {v6.8h}, [x1], #16 + st1 {v7.8h}, [x11], #16 + st1 {v22.4s,v23.4s}, [x0], #32 + st1 {v24.4s,v25.4s}, [x10], #32 + + subs w5, w5, #8 + b.le 9f + tst w7, #2 // LR_HAVE_RIGHT + mov v0.16b, v1.16b + mov v16.16b, v17.16b + ld1 {v1.8h}, [x3], #16 + ld1 {v17.8h}, [x12], #16 + mov v2.16b, v4.16b + umull2 v3.4s, v0.8h, v0.8h + umull v4.4s, v1.4h, v1.4h + mov v18.16b, v20.16b + umull2 v19.4s, v16.8h, v16.8h + umull v20.4s, v17.4h, v17.4h + + b.ne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +5: // Produce 4 pixels, 6 <= w < 10 + add3 4, .4h + st1 {v6.4h}, [x1], #8 + st1 {v7.4h}, [x11], #8 + st1 {v22.4s}, [x0], #16 + st1 {v24.4s}, [x10], #16 + + subs w5, w5, #4 // 2 <= w < 6 + ext v0.16b, v0.16b, v1.16b, #8 + ext v16.16b, v16.16b, v17.16b, #8 + +6: // Pad the right edge and produce the last few pixels. + // 2 <= w < 6, 2-5 pixels valid in v0 + sub w13, w5, #2 + // w13 = (pixels valid - 2) + adr x14, L(box3_variable_shift_tbl) + ldrh w13, [x14, w13, uxtw #1] + sub x13, x14, w13, uxth + br x13 + // Shift v0 right, shifting out invalid pixels, + // shift v0 left to the original offset, shifting in padding pixels. +22: // 2 pixels valid + ext v0.16b, v0.16b, v0.16b, #4 + ext v16.16b, v16.16b, v16.16b, #4 + ext v0.16b, v0.16b, v30.16b, #12 + ext v16.16b, v16.16b, v31.16b, #12 + b 88f +33: // 3 pixels valid + ext v0.16b, v0.16b, v0.16b, #6 + ext v16.16b, v16.16b, v16.16b, #6 + ext v0.16b, v0.16b, v30.16b, #10 + ext v16.16b, v16.16b, v31.16b, #10 + b 88f +44: // 4 pixels valid + ext v0.16b, v0.16b, v0.16b, #8 + ext v16.16b, v16.16b, v16.16b, #8 + ext v0.16b, v0.16b, v30.16b, #8 + ext v16.16b, v16.16b, v31.16b, #8 + b 88f +55: // 5 pixels valid + ext v0.16b, v0.16b, v0.16b, #10 + ext v16.16b, v16.16b, v16.16b, #10 + ext v0.16b, v0.16b, v30.16b, #6 + ext v16.16b, v16.16b, v31.16b, #6 + b 88f + +L(box3_variable_shift_tbl): + .hword L(box3_variable_shift_tbl) - 22b + .hword L(box3_variable_shift_tbl) - 33b + .hword L(box3_variable_shift_tbl) - 44b + .hword L(box3_variable_shift_tbl) - 55b + +88: + umull v2.4s, v0.4h, v0.4h + umull2 v3.4s, v0.8h, v0.8h + umull v18.4s, v16.4h, v16.4h + umull2 v19.4s, v16.8h, v16.8h + + add3 4, .4h + subs w5, w5, #4 + st1 {v6.4h}, [x1], #8 + st1 {v7.4h}, [x11], #8 + st1 {v22.4s}, [x0], #16 + st1 {v24.4s}, [x10], #16 + b.le 9f + ext v0.16b, v0.16b, v0.16b, #8 + ext v16.16b, v16.16b, v16.16b, #8 + mov v2.16b, v3.16b + mov v3.16b, v4.16b + mov v18.16b, v19.16b + mov v19.16b, v20.16b + // Only one needed pixel left, but do a normal 4 pixel + // addition anyway + add3 4, .4h + st1 {v6.4h}, [x1], #8 + st1 {v7.4h}, [x11], #8 + st1 {v22.4s}, [x0], #16 + st1 {v24.4s}, [x10], #16 + +9: + subs w6, w6, #2 + b.le 0f + // Jump to the next row and loop horizontally + add x0, x0, x9, lsl #1 + add x10, x10, x9, lsl #1 + add x1, x1, x9 + add x11, x11, x9 + add x3, x3, x4 + add x12, x12, x4 + mov w5, w8 + b 1b +0: + ret +.purgem add3 +endfunc + +// void dav1d_sgr_box5_h_16bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_h_16bpc_neon, export=1 + add w5, w5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add x10, x0, #(4*SUM_STRIDE) // sumsq + add x11, x1, #(2*SUM_STRIDE) // sum + add x12, x3, x4 // src + lsl x4, x4, #1 + mov x9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + // With LR_HAVE_RIGHT, align to 8, without it, align to 4. + // Subtract the number of pixels read from the input from the stride. + tst w7, #2 // LR_HAVE_RIGHT + b.ne 0f + // !LR_HAVE_RIGHT + add w13, w5, #3 + bic w13, w13, #3 + add w14, w5, #13 + b 1f +0: + add w13, w5, #7 + bic w13, w13, #7 + add w14, w5, #15 +1: + sub x9, x9, w13, uxtw #1 + bic w14, w14, #7 + sub x4, x4, w14, uxtw #1 + + // Store the width for the vertical loop + mov w8, w5 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 2f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #6 + sub x12, x12, #6 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add x4, x4, #6 + +1: // Loop vertically + ld1 {v0.8h, v1.8h}, [x3], #32 + ld1 {v16.8h, v17.8h}, [x12], #32 + + tst w7, #1 // LR_HAVE_LEFT + b.eq 0f + cbz x2, 2f + // LR_HAVE_LEFT, left != NULL + ld1 {v2.d}[1], [x2], #8 + // Move x3/x12 back to account for the last 3 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #6 + sub x12, x12, #6 + ld1 {v18.d}[1], [x2], #8 + ext v1.16b, v0.16b, v1.16b, #10 + ext v0.16b, v2.16b, v0.16b, #10 + ext v17.16b, v16.16b, v17.16b, #10 + ext v16.16b, v18.16b, v16.16b, #10 + b 2f +0: + // !LR_HAVE_LEFT, fill v2 with the leftmost pixel + // and shift v0/v1 to have 3x the first pixel at the front. + dup v2.8h, v0.h[0] + dup v18.8h, v16.h[0] + // Move x3 back to account for the last 6 bytes we loaded before, + // which we shifted out. + sub x3, x3, #6 + sub x12, x12, #6 + ext v1.16b, v0.16b, v1.16b, #10 + ext v0.16b, v2.16b, v0.16b, #10 + ext v17.16b, v16.16b, v17.16b, #10 + ext v16.16b, v18.16b, v16.16b, #10 + +2: + umull v2.4s, v0.4h, v0.4h + umull2 v3.4s, v0.8h, v0.8h + umull v4.4s, v1.4h, v1.4h + umull v18.4s, v16.4h, v16.4h + umull2 v19.4s, v16.8h, v16.8h + umull v20.4s, v17.4h, v17.4h + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub w13, w5, #(2 + 16 - 3 + 1) + ldr h30, [x3, w13, sxtw #1] + ldr h31, [x12, w13, sxtw #1] + // Fill v30/v31 with the right padding pixel + dup v30.8h, v30.h[0] + dup v31.8h, v31.h[0] +3: // !LR_HAVE_RIGHT + // If we'll have to pad the right edge we need to quit early here. + cmp w5, #11 + b.ge 4f // If w >= 11, all used input pixels are valid + cmp w5, #7 + b.ge 5f // If w >= 7, we can produce 4 pixels + b 6f + +4: // Loop horizontally +.macro add5 w, wd + ext v24.16b, v0.16b, v1.16b, #2 + ext v25.16b, v0.16b, v1.16b, #4 + ext v26.16b, v0.16b, v1.16b, #6 + ext v27.16b, v0.16b, v1.16b, #8 + + add v6\wd, v0\wd, v24\wd + add v25\wd, v25\wd, v26\wd + add v6\wd, v6\wd, v27\wd + + ext v26.16b, v16.16b, v17.16b, #2 + ext v27.16b, v16.16b, v17.16b, #4 + ext v28.16b, v16.16b, v17.16b, #6 + ext v29.16b, v16.16b, v17.16b, #8 + + add v7\wd, v16\wd, v26\wd + add v27\wd, v27\wd, v28\wd + add v7\wd, v7\wd, v29\wd + add v6\wd, v6\wd, v25\wd + add v7\wd, v7\wd, v27\wd + + ext_n v24.16b, v25.16b, v2.16b, v3.16b, v4.16b, #4, \w + ext_n v26.16b, v27.16b, v2.16b, v3.16b, v4.16b, #8, \w + ext_n v28.16b, v29.16b, v2.16b, v3.16b, v4.16b, #12, \w + + add_n v22.4s, v23.4s, v2.4s, v3.4s, v24.4s, v25.4s, \w + add_n v26.4s, v27.4s, v26.4s, v27.4s, v28.4s, v29.4s, \w + add_n v22.4s, v23.4s, v22.4s, v23.4s, v3.4s, v4.4s, \w + add_n v22.4s, v23.4s, v22.4s, v23.4s, v26.4s, v27.4s, \w + + ext_n v24.16b, v25.16b, v18.16b, v19.16b, v20.16b, #4, \w + ext_n v26.16b, v27.16b, v18.16b, v19.16b, v20.16b, #8, \w + ext_n v28.16b, v29.16b, v18.16b, v19.16b, v20.16b, #12, \w + + add_n v24.4s, v25.4s, v18.4s, v19.4s, v24.4s, v25.4s, \w + add_n v26.4s, v27.4s, v26.4s, v27.4s, v28.4s, v29.4s, \w + add_n v24.4s, v25.4s, v24.4s, v25.4s, v19.4s, v20.4s, \w + add_n v24.4s, v25.4s, v24.4s, v25.4s, v26.4s, v27.4s, \w +.endm + add5 8, .8h + st1 {v6.8h}, [x1], #16 + st1 {v7.8h}, [x11], #16 + st1 {v22.4s,v23.4s}, [x0], #32 + st1 {v24.4s,v25.4s}, [x10], #32 + + subs w5, w5, #8 + b.le 9f + tst w7, #2 // LR_HAVE_RIGHT + mov v0.16b, v1.16b + mov v16.16b, v17.16b + ld1 {v1.8h}, [x3], #16 + ld1 {v17.8h}, [x12], #16 + mov v2.16b, v4.16b + umull2 v3.4s, v0.8h, v0.8h + umull v4.4s, v1.4h, v1.4h + mov v18.16b, v20.16b + umull2 v19.4s, v16.8h, v16.8h + umull v20.4s, v17.4h, v17.4h + + b.ne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +5: // Produce 4 pixels, 7 <= w < 11 + add5 4, .4h + st1 {v6.4h}, [x1], #8 + st1 {v7.4h}, [x11], #8 + st1 {v22.4s}, [x0], #16 + st1 {v24.4s}, [x10], #16 + + subs w5, w5, #4 // 3 <= w < 7 + ext v0.16b, v0.16b, v1.16b, #8 + ext v16.16b, v16.16b, v17.16b, #8 + +6: // Pad the right edge and produce the last few pixels. + // w < 7, w+1 pixels valid in v0/v4 + sub w13, w5, #1 + // w13 = pixels valid - 2 + adr x14, L(box5_variable_shift_tbl) + ldrh w13, [x14, w13, uxtw #1] + mov v1.16b, v30.16b + mov v17.16b, v31.16b + sub x13, x14, w13, uxth + br x13 + // Shift v0 right, shifting out invalid pixels, + // shift v0 left to the original offset, shifting in padding pixels. +22: // 2 pixels valid + ext v0.16b, v0.16b, v0.16b, #4 + ext v16.16b, v16.16b, v16.16b, #4 + ext v0.16b, v0.16b, v30.16b, #12 + ext v16.16b, v16.16b, v31.16b, #12 + b 88f +33: // 3 pixels valid + ext v0.16b, v0.16b, v0.16b, #6 + ext v16.16b, v16.16b, v16.16b, #6 + ext v0.16b, v0.16b, v30.16b, #10 + ext v16.16b, v16.16b, v31.16b, #10 + b 88f +44: // 4 pixels valid + ext v0.16b, v0.16b, v0.16b, #8 + ext v16.16b, v16.16b, v16.16b, #8 + ext v0.16b, v0.16b, v30.16b, #8 + ext v16.16b, v16.16b, v31.16b, #8 + b 88f +55: // 5 pixels valid + ext v0.16b, v0.16b, v0.16b, #10 + ext v16.16b, v16.16b, v16.16b, #10 + ext v0.16b, v0.16b, v30.16b, #6 + ext v16.16b, v16.16b, v31.16b, #6 + b 88f +66: // 6 pixels valid + ext v0.16b, v0.16b, v0.16b, #12 + ext v16.16b, v16.16b, v16.16b, #12 + ext v0.16b, v0.16b, v30.16b, #4 + ext v16.16b, v16.16b, v31.16b, #4 + b 88f +77: // 7 pixels valid + ext v0.16b, v0.16b, v0.16b, #14 + ext v16.16b, v16.16b, v16.16b, #14 + ext v0.16b, v0.16b, v30.16b, #2 + ext v16.16b, v16.16b, v31.16b, #2 + b 88f + +L(box5_variable_shift_tbl): + .hword L(box5_variable_shift_tbl) - 22b + .hword L(box5_variable_shift_tbl) - 33b + .hword L(box5_variable_shift_tbl) - 44b + .hword L(box5_variable_shift_tbl) - 55b + .hword L(box5_variable_shift_tbl) - 66b + .hword L(box5_variable_shift_tbl) - 77b + +88: + umull v2.4s, v0.4h, v0.4h + umull2 v3.4s, v0.8h, v0.8h + umull v4.4s, v1.4h, v1.4h + umull v18.4s, v16.4h, v16.4h + umull2 v19.4s, v16.8h, v16.8h + umull v20.4s, v17.4h, v17.4h + + add5 4, .4h + subs w5, w5, #4 + st1 {v6.4h}, [x1], #8 + st1 {v7.4h}, [x11], #8 + st1 {v22.4s}, [x0], #16 + st1 {v24.4s}, [x10], #16 + b.le 9f + ext v0.16b, v0.16b, v1.16b, #8 + ext v16.16b, v16.16b, v17.16b, #8 + mov v2.16b, v3.16b + mov v3.16b, v4.16b + mov v18.16b, v19.16b + mov v19.16b, v20.16b + add5 4, .4h + st1 {v6.4h}, [x1], #8 + st1 {v7.4h}, [x11], #8 + st1 {v22.4s}, [x0], #16 + st1 {v24.4s}, [x10], #16 + +9: + subs w6, w6, #2 + b.le 0f + // Jump to the next row and loop horizontally + add x0, x0, x9, lsl #1 + add x10, x10, x9, lsl #1 + add x1, x1, x9 + add x11, x11, x9 + add x3, x3, x4 + add x12, x12, x4 + mov w5, w8 + b 1b +0: + ret +.purgem add5 +endfunc + +sgr_funcs 16 diff --git a/third_party/dav1d/src/arm/64/looprestoration_common.S b/third_party/dav1d/src/arm/64/looprestoration_common.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/looprestoration_common.S @@ -0,0 +1,432 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define SUM_STRIDE (384+16) + +// void dav1d_sgr_box3_v_neon(int32_t *sumsq, int16_t *sum, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_v_neon, export=1 + add w10, w3, #2 // Number of output rows to move back + mov w11, w3 // Number of input rows to move back + add w2, w2, #2 // Actual summed width + mov x7, #(4*SUM_STRIDE) // sumsq stride + mov x8, #(2*SUM_STRIDE) // sum stride + sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride + sub x1, x1, #(2*SUM_STRIDE) // sum -= stride + + tst w4, #4 // LR_HAVE_TOP + b.eq 0f + // If have top, read from row -2. + sub x5, x0, #(4*SUM_STRIDE) + sub x6, x1, #(2*SUM_STRIDE) + add w11, w11, #2 + b 1f +0: + // !LR_HAVE_TOP + // If we don't have top, read from row 0 even if + // we start writing to row -1. + add x5, x0, #(4*SUM_STRIDE) + add x6, x1, #(2*SUM_STRIDE) +1: + + tst w4, #8 // LR_HAVE_BOTTOM + b.eq 1f + // LR_HAVE_BOTTOM + add w3, w3, #2 // Sum all h+2 lines with the main loop + add w11, w11, #2 +1: + mov w9, w3 // Backup of h for next loops + +1: + // Start of horizontal loop; start one vertical filter slice. + // Start loading rows into v16-v21 and v24-v26 taking top + // padding into consideration. + tst w4, #4 // LR_HAVE_TOP + ld1 {v16.4s, v17.4s}, [x5], x7 + ld1 {v24.8h}, [x6], x8 + b.eq 2f + // LR_HAVE_TOP + ld1 {v18.4s, v19.4s}, [x5], x7 + ld1 {v25.8h}, [x6], x8 + ld1 {v20.4s, v21.4s}, [x5], x7 + ld1 {v26.8h}, [x6], x8 + b 3f +2: // !LR_HAVE_TOP + mov v18.16b, v16.16b + mov v19.16b, v17.16b + mov v25.16b, v24.16b + mov v20.16b, v16.16b + mov v21.16b, v17.16b + mov v26.16b, v24.16b + +3: + subs w3, w3, #1 +.macro add3 + add v16.4s, v16.4s, v18.4s + add v17.4s, v17.4s, v19.4s + add v24.8h, v24.8h, v25.8h + add v16.4s, v16.4s, v20.4s + add v17.4s, v17.4s, v21.4s + add v24.8h, v24.8h, v26.8h + st1 {v16.4s, v17.4s}, [x0], x7 + st1 {v24.8h}, [x1], x8 +.endm + add3 + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v24.16b, v25.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v25.16b, v26.16b + b.le 4f + ld1 {v20.4s, v21.4s}, [x5], x7 + ld1 {v26.8h}, [x6], x8 + b 3b + +4: + tst w4, #8 // LR_HAVE_BOTTOM + b.ne 5f + // !LR_HAVE_BOTTOM + // Produce two more rows, extending the already loaded rows. + add3 + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v24.16b, v25.16b + add3 + +5: // End of one vertical slice. + subs w2, w2, #8 + b.le 0f + // Move pointers back up to the top and loop horizontally. + // Input pointers + msub x5, x7, x11, x5 + msub x6, x8, x11, x6 + // Output pointers + msub x0, x7, x10, x0 + msub x1, x8, x10, x1 + add x0, x0, #32 + add x1, x1, #16 + add x5, x5, #32 + add x6, x6, #16 + mov w3, w9 + b 1b + +0: + ret +.purgem add3 +endfunc + +// void dav1d_sgr_box5_v_neon(int32_t *sumsq, int16_t *sum, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_v_neon, export=1 + add w10, w3, #2 // Number of output rows to move back + mov w11, w3 // Number of input rows to move back + add w2, w2, #8 // Actual summed width + mov x7, #(4*SUM_STRIDE) // sumsq stride + mov x8, #(2*SUM_STRIDE) // sum stride + sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride + sub x1, x1, #(2*SUM_STRIDE) // sum -= stride + + tst w4, #4 // LR_HAVE_TOP + b.eq 0f + // If have top, read from row -2. + sub x5, x0, #(4*SUM_STRIDE) + sub x6, x1, #(2*SUM_STRIDE) + add w11, w11, #2 + b 1f +0: + // !LR_HAVE_TOP + // If we don't have top, read from row 0 even if + // we start writing to row -1. + add x5, x0, #(4*SUM_STRIDE) + add x6, x1, #(2*SUM_STRIDE) +1: + + tst w4, #8 // LR_HAVE_BOTTOM + b.eq 0f + // LR_HAVE_BOTTOM + add w3, w3, #2 // Handle h+2 lines with the main loop + add w11, w11, #2 + b 1f +0: + // !LR_HAVE_BOTTOM + sub w3, w3, #1 // Handle h-1 lines with the main loop +1: + mov w9, w3 // Backup of h for next loops + +1: + // Start of horizontal loop; start one vertical filter slice. + // Start loading rows into v16-v25 and v26-v30 taking top + // padding into consideration. + tst w4, #4 // LR_HAVE_TOP + ld1 {v16.4s, v17.4s}, [x5], x7 + ld1 {v26.8h}, [x6], x8 + b.eq 2f + // LR_HAVE_TOP + ld1 {v20.4s, v21.4s}, [x5], x7 + ld1 {v28.8h}, [x6], x8 + mov v18.16b, v16.16b + mov v19.16b, v17.16b + mov v27.16b, v26.16b + ld1 {v22.4s, v23.4s}, [x5], x7 + ld1 {v29.8h}, [x6], x8 + b 3f +2: // !LR_HAVE_TOP + mov v18.16b, v16.16b + mov v19.16b, v17.16b + mov v27.16b, v26.16b + mov v20.16b, v16.16b + mov v21.16b, v17.16b + mov v28.16b, v26.16b + mov v22.16b, v16.16b + mov v23.16b, v17.16b + mov v29.16b, v26.16b + +3: + cbz w3, 4f + ld1 {v24.4s, v25.4s}, [x5], x7 + ld1 {v30.8h}, [x6], x8 + +3: + // Start of vertical loop + subs w3, w3, #2 +.macro add5 + add v16.4s, v16.4s, v18.4s + add v17.4s, v17.4s, v19.4s + add v26.8h, v26.8h, v27.8h + add v0.4s, v20.4s, v22.4s + add v1.4s, v21.4s, v23.4s + add v2.8h, v28.8h, v29.8h + add v16.4s, v16.4s, v24.4s + add v17.4s, v17.4s, v25.4s + add v26.8h, v26.8h, v30.8h + add v16.4s, v16.4s, v0.4s + add v17.4s, v17.4s, v1.4s + add v26.8h, v26.8h, v2.8h + st1 {v16.4s, v17.4s}, [x0], x7 + st1 {v26.8h}, [x1], x8 +.endm + add5 +.macro shift2 + mov v16.16b, v20.16b + mov v17.16b, v21.16b + mov v26.16b, v28.16b + mov v18.16b, v22.16b + mov v19.16b, v23.16b + mov v27.16b, v29.16b + mov v20.16b, v24.16b + mov v21.16b, v25.16b + mov v28.16b, v30.16b +.endm + shift2 + add x0, x0, x7 + add x1, x1, x8 + b.le 5f + ld1 {v22.4s, v23.4s}, [x5], x7 + ld1 {v29.8h}, [x6], x8 + ld1 {v24.4s, v25.4s}, [x5], x7 + ld1 {v30.8h}, [x6], x8 + b 3b + +4: + // h == 1, !LR_HAVE_BOTTOM. + // Pad the last row with the only content row, and add. + mov v24.16b, v22.16b + mov v25.16b, v23.16b + mov v30.16b, v29.16b + add5 + shift2 + add x0, x0, x7 + add x1, x1, x8 + add5 + b 6f + +5: + tst w4, #8 // LR_HAVE_BOTTOM + b.ne 6f + // !LR_HAVE_BOTTOM + cbnz w3, 5f + // The intended three edge rows left; output the one at h-2 and + // the past edge one at h. + ld1 {v22.4s, v23.4s}, [x5], x7 + ld1 {v29.8h}, [x6], x8 + // Pad the past-edge row from the last content row. + mov v24.16b, v22.16b + mov v25.16b, v23.16b + mov v30.16b, v29.16b + add5 + shift2 + add x0, x0, x7 + add x1, x1, x8 + // The last two rows are already padded properly here. + add5 + b 6f + +5: + // w3 == -1, two rows left, output one. + // Pad the last two rows from the mid one. + mov v22.16b, v20.16b + mov v23.16b, v21.16b + mov v29.16b, v28.16b + mov v24.16b, v20.16b + mov v25.16b, v21.16b + mov v30.16b, v28.16b + add5 + add x0, x0, x7 + add x1, x1, x8 + b 6f + +6: // End of one vertical slice. + subs w2, w2, #8 + b.le 0f + // Move pointers back up to the top and loop horizontally. + // Input pointers + msub x5, x7, x11, x5 + msub x6, x8, x11, x6 + // Output pointers + msub x0, x7, x10, x0 + msub x1, x8, x10, x1 + add x0, x0, #32 + add x1, x1, #16 + add x5, x5, #32 + add x6, x6, #16 + mov w3, w9 + b 1b + +0: + ret +.purgem add5 +endfunc + +// void dav1d_sgr_calc_ab1_neon(int32_t *a, int16_t *b, +// const int w, const int h, const int strength, +// const int bitdepth_max); +// void dav1d_sgr_calc_ab2_neon(int32_t *a, int16_t *b, +// const int w, const int h, const int strength, +// const int bitdepth_max); +function sgr_calc_ab1_neon, export=1 + clz w9, w5 + add x3, x3, #2 // h += 2 + movi v31.4s, #9 // n + mov x5, #455 + mov x8, #SUM_STRIDE + b sgr_calc_ab_neon +endfunc + +function sgr_calc_ab2_neon, export=1 + clz w9, w5 + add x3, x3, #3 // h += 3 + asr x3, x3, #1 // h /= 2 + movi v31.4s, #25 // n + mov x5, #164 + mov x8, #(2*SUM_STRIDE) +endfunc + +function sgr_calc_ab_neon + sub w9, w9, #24 // -bitdepth_min_8 + movrel x12, X(sgr_x_by_x) + ld1 {v16.16b, v17.16b, v18.16b}, [x12] + dup v6.8h, w9 // -bitdepth_min_8 + movi v19.16b, #5 + movi v20.8b, #55 // idx of last 5 + movi v21.8b, #72 // idx of last 4 + movi v22.8b, #101 // idx of last 3 + movi v23.8b, #169 // idx of last 2 + movi v24.8b, #254 // idx of last 1 + saddl v7.4s, v6.4h, v6.4h // -2*bitdepth_min_8 + add x2, x2, #2 // w += 2 + add x7, x2, #7 + bic x7, x7, #7 // aligned w + sub x7, x8, x7 // increment between rows + movi v29.8h, #1, lsl #8 + dup v28.4s, w4 + dup v30.4s, w5 // one_by_x + sub x0, x0, #(4*(SUM_STRIDE)) + sub x1, x1, #(2*(SUM_STRIDE)) + mov x6, x2 // backup of w + sub v16.16b, v16.16b, v19.16b + sub v17.16b, v17.16b, v19.16b + sub v18.16b, v18.16b, v19.16b +1: + subs x2, x2, #8 + ld1 {v0.4s, v1.4s}, [x0] // a + ld1 {v2.8h}, [x1] // b + srshl v0.4s, v0.4s, v7.4s + srshl v1.4s, v1.4s, v7.4s + srshl v4.8h, v2.8h, v6.8h + mul v0.4s, v0.4s, v31.4s // a * n + mul v1.4s, v1.4s, v31.4s // a * n + umull v3.4s, v4.4h, v4.4h // b * b + umull2 v4.4s, v4.8h, v4.8h // b * b + uqsub v0.4s, v0.4s, v3.4s // imax(a * n - b * b, 0) + uqsub v1.4s, v1.4s, v4.4s // imax(a * n - b * b, 0) + mul v0.4s, v0.4s, v28.4s // p * s + mul v1.4s, v1.4s, v28.4s // p * s + uqshrn v0.4h, v0.4s, #16 + uqshrn2 v0.8h, v1.4s, #16 + uqrshrn v0.8b, v0.8h, #4 // imin(z, 255) + + cmhi v25.8b, v0.8b, v20.8b // = -1 if sgr_x_by_x[v0] < 5 + cmhi v26.8b, v0.8b, v21.8b // = -1 if sgr_x_by_x[v0] < 4 + tbl v1.8b, {v16.16b, v17.16b, v18.16b}, v0.8b + cmhi v27.8b, v0.8b, v22.8b // = -1 if sgr_x_by_x[v0] < 3 + cmhi v4.8b, v0.8b, v23.8b // = -1 if sgr_x_by_x[v0] < 2 + add v25.8b, v25.8b, v26.8b + cmhi v5.8b, v0.8b, v24.8b // = -1 if sgr_x_by_x[v0] < 1 + add v27.8b, v27.8b, v4.8b + add v5.8b, v5.8b, v19.8b + add v25.8b, v25.8b, v27.8b + add v1.8b, v1.8b, v5.8b + add v1.8b, v1.8b, v25.8b + uxtl v1.8h, v1.8b // x + + umull v3.4s, v1.4h, v2.4h // x * BB[i] + umull2 v4.4s, v1.8h, v2.8h // x * BB[i] + mul v3.4s, v3.4s, v30.4s // x * BB[i] * sgr_one_by_x + mul v4.4s, v4.4s, v30.4s // x * BB[i] * sgr_one_by_x + srshr v3.4s, v3.4s, #12 // AA[i] + srshr v4.4s, v4.4s, #12 // AA[i] + sub v2.8h, v29.8h, v1.8h // 256 - x + + st1 {v3.4s, v4.4s}, [x0], #32 + st1 {v2.8h}, [x1], #16 + b.gt 1b + + subs x3, x3, #1 + b.le 0f + add x0, x0, x7, lsl #2 + add x1, x1, x7, lsl #1 + mov x2, x6 + b 1b +0: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/looprestoration_tmpl.S b/third_party/dav1d/src/arm/64/looprestoration_tmpl.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/looprestoration_tmpl.S @@ -0,0 +1,597 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" + +#define FILTER_OUT_STRIDE 384 + +.macro sgr_funcs bpc +// void dav1d_sgr_finish_filter1_Xbpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter1_\bpc\()bpc_neon, export=1 + sub x7, x3, #(4*SUM_STRIDE) + add x8, x3, #(4*SUM_STRIDE) + sub x9, x4, #(2*SUM_STRIDE) + add x10, x4, #(2*SUM_STRIDE) + mov x11, #SUM_STRIDE + mov x12, #FILTER_OUT_STRIDE + add x13, x5, #7 + bic x13, x13, #7 // Aligned width +.if \bpc == 8 + sub x2, x2, x13 +.else + sub x2, x2, x13, lsl #1 +.endif + sub x12, x12, x13 + sub x11, x11, x13 + sub x11, x11, #4 // We read 4 extra elements from a + sub x14, x11, #4 // We read 8 extra elements from b + mov x13, x5 + movi v6.8h, #3 + movi v7.4s, #3 +1: + ld1 {v0.8h, v1.8h}, [x9], #32 + ld1 {v2.8h, v3.8h}, [x4], #32 + ld1 {v4.8h, v5.8h}, [x10], #32 + ld1 {v16.4s, v17.4s, v18.4s}, [x7], #48 + ld1 {v19.4s, v20.4s, v21.4s}, [x3], #48 + ld1 {v22.4s, v23.4s, v24.4s}, [x8], #48 + +2: + subs x5, x5, #8 + ext v25.16b, v0.16b, v1.16b, #2 // -stride + ext v26.16b, v2.16b, v3.16b, #2 // 0 + ext v27.16b, v4.16b, v5.16b, #2 // +stride + ext v28.16b, v0.16b, v1.16b, #4 // +1-stride + ext v29.16b, v2.16b, v3.16b, #4 // +1 + ext v30.16b, v4.16b, v5.16b, #4 // +1+stride + add v2.8h, v2.8h, v25.8h // -1, -stride + add v26.8h, v26.8h, v27.8h // 0, +stride + add v0.8h, v0.8h, v28.8h // -1-stride, +1-stride + add v2.8h, v2.8h, v26.8h + add v4.8h, v4.8h, v30.8h // -1+stride, +1+stride + add v2.8h, v2.8h, v29.8h // +1 + add v0.8h, v0.8h, v4.8h + + ext v25.16b, v16.16b, v17.16b, #4 // -stride + ext v26.16b, v17.16b, v18.16b, #4 + shl v2.8h, v2.8h, #2 + ext v27.16b, v16.16b, v17.16b, #8 // +1-stride + ext v28.16b, v17.16b, v18.16b, #8 + ext v29.16b, v19.16b, v20.16b, #4 // 0 + ext v30.16b, v20.16b, v21.16b, #4 + mla v2.8h, v0.8h, v6.8h // * 3 -> a + add v25.4s, v25.4s, v19.4s // -stride, -1 + add v26.4s, v26.4s, v20.4s + add v16.4s, v16.4s, v27.4s // -1-stride, +1-stride + add v17.4s, v17.4s, v28.4s + ext v27.16b, v19.16b, v20.16b, #8 // +1 + ext v28.16b, v20.16b, v21.16b, #8 + add v16.4s, v16.4s, v22.4s // -1+stride + add v17.4s, v17.4s, v23.4s + add v29.4s, v29.4s, v27.4s // 0, +1 + add v30.4s, v30.4s, v28.4s + add v25.4s, v25.4s, v29.4s + add v26.4s, v26.4s, v30.4s + ext v27.16b, v22.16b, v23.16b, #4 // +stride + ext v28.16b, v23.16b, v24.16b, #4 + ext v29.16b, v22.16b, v23.16b, #8 // +1+stride + ext v30.16b, v23.16b, v24.16b, #8 +.if \bpc == 8 + ld1 {v19.8b}, [x1], #8 // src +.else + ld1 {v19.8h}, [x1], #16 // src +.endif + add v25.4s, v25.4s, v27.4s // +stride + add v26.4s, v26.4s, v28.4s + add v16.4s, v16.4s, v29.4s // +1+stride + add v17.4s, v17.4s, v30.4s + shl v25.4s, v25.4s, #2 + shl v26.4s, v26.4s, #2 + mla v25.4s, v16.4s, v7.4s // * 3 -> b + mla v26.4s, v17.4s, v7.4s +.if \bpc == 8 + uxtl v19.8h, v19.8b // src +.endif + mov v0.16b, v1.16b + umlal v25.4s, v2.4h, v19.4h // b + a * src + umlal2 v26.4s, v2.8h, v19.8h + mov v2.16b, v3.16b + rshrn v25.4h, v25.4s, #9 + rshrn2 v25.8h, v26.4s, #9 + mov v4.16b, v5.16b + st1 {v25.8h}, [x0], #16 + + b.le 3f + mov v16.16b, v18.16b + mov v19.16b, v21.16b + mov v22.16b, v24.16b + ld1 {v1.8h}, [x9], #16 + ld1 {v3.8h}, [x4], #16 + ld1 {v5.8h}, [x10], #16 + ld1 {v17.4s, v18.4s}, [x7], #32 + ld1 {v20.4s, v21.4s}, [x3], #32 + ld1 {v23.4s, v24.4s}, [x8], #32 + b 2b + +3: + subs x6, x6, #1 + b.le 0f + mov x5, x13 + add x0, x0, x12, lsl #1 + add x1, x1, x2 + add x3, x3, x11, lsl #2 + add x7, x7, x11, lsl #2 + add x8, x8, x11, lsl #2 + add x4, x4, x14, lsl #1 + add x9, x9, x14, lsl #1 + add x10, x10, x14, lsl #1 + b 1b +0: + ret +endfunc + +// void dav1d_sgr_finish_filter2_Xbpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter2_\bpc\()bpc_neon, export=1 + add x7, x3, #(4*(SUM_STRIDE)) + sub x3, x3, #(4*(SUM_STRIDE)) + add x8, x4, #(2*(SUM_STRIDE)) + sub x4, x4, #(2*(SUM_STRIDE)) + mov x9, #(2*SUM_STRIDE) + mov x10, #FILTER_OUT_STRIDE + add x11, x5, #7 + bic x11, x11, #7 // Aligned width +.if \bpc == 8 + sub x2, x2, x11 +.else + sub x2, x2, x11, lsl #1 +.endif + sub x10, x10, x11 + sub x9, x9, x11 + sub x9, x9, #4 // We read 4 extra elements from a + sub x12, x9, #4 // We read 8 extra elements from b + mov x11, x5 + movi v4.8h, #5 + movi v5.4s, #5 + movi v6.8h, #6 + movi v7.4s, #6 +1: + ld1 {v0.8h, v1.8h}, [x4], #32 + ld1 {v2.8h, v3.8h}, [x8], #32 + ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 + ld1 {v19.4s, v20.4s, v21.4s}, [x7], #48 + +2: + subs x5, x5, #8 + ext v24.16b, v0.16b, v1.16b, #4 // +1-stride + ext v25.16b, v2.16b, v3.16b, #4 // +1+stride + ext v22.16b, v0.16b, v1.16b, #2 // -stride + ext v23.16b, v2.16b, v3.16b, #2 // +stride + add v0.8h, v0.8h, v24.8h // -1-stride, +1-stride + add v25.8h, v2.8h, v25.8h // -1+stride, +1+stride + add v2.8h, v22.8h, v23.8h // -stride, +stride + add v0.8h, v0.8h, v25.8h + + ext v22.16b, v16.16b, v17.16b, #4 // -stride + ext v23.16b, v17.16b, v18.16b, #4 + ext v24.16b, v19.16b, v20.16b, #4 // +stride + ext v25.16b, v20.16b, v21.16b, #4 + ext v26.16b, v16.16b, v17.16b, #8 // +1-stride + ext v27.16b, v17.16b, v18.16b, #8 + ext v28.16b, v19.16b, v20.16b, #8 // +1+stride + ext v29.16b, v20.16b, v21.16b, #8 + mul v0.8h, v0.8h, v4.8h // * 5 + mla v0.8h, v2.8h, v6.8h // * 6 +.if \bpc == 8 + ld1 {v31.8b}, [x1], #8 +.else + ld1 {v31.8h}, [x1], #16 +.endif + add v16.4s, v16.4s, v26.4s // -1-stride, +1-stride + add v17.4s, v17.4s, v27.4s + add v19.4s, v19.4s, v28.4s // -1+stride, +1+stride + add v20.4s, v20.4s, v29.4s + add v16.4s, v16.4s, v19.4s + add v17.4s, v17.4s, v20.4s + + add v22.4s, v22.4s, v24.4s // -stride, +stride + add v23.4s, v23.4s, v25.4s + // This is, surprisingly, faster than other variants where the + // mul+mla pairs are further apart, on Cortex A53. + mul v16.4s, v16.4s, v5.4s // * 5 + mla v16.4s, v22.4s, v7.4s // * 6 + mul v17.4s, v17.4s, v5.4s // * 5 + mla v17.4s, v23.4s, v7.4s // * 6 + +.if \bpc == 8 + uxtl v31.8h, v31.8b +.endif + umlal v16.4s, v0.4h, v31.4h // b + a * src + umlal2 v17.4s, v0.8h, v31.8h + mov v0.16b, v1.16b + rshrn v16.4h, v16.4s, #9 + rshrn2 v16.8h, v17.4s, #9 + mov v2.16b, v3.16b + st1 {v16.8h}, [x0], #16 + + b.le 3f + mov v16.16b, v18.16b + mov v19.16b, v21.16b + ld1 {v1.8h}, [x4], #16 + ld1 {v3.8h}, [x8], #16 + ld1 {v17.4s, v18.4s}, [x3], #32 + ld1 {v20.4s, v21.4s}, [x7], #32 + b 2b + +3: + subs x6, x6, #1 + b.le 0f + mov x5, x11 + add x0, x0, x10, lsl #1 + add x1, x1, x2 + add x3, x3, x9, lsl #2 + add x7, x7, x9, lsl #2 + add x4, x4, x12, lsl #1 + add x8, x8, x12, lsl #1 + mov x13, x3 + mov x14, x4 + + ld1 {v0.8h, v1.8h}, [x4], #32 + ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 + +4: + subs x5, x5, #8 + ext v23.16b, v0.16b, v1.16b, #4 // +1 + ext v22.16b, v0.16b, v1.16b, #2 // 0 + add v0.8h, v0.8h, v23.8h // -1, +1 + + ext v24.16b, v16.16b, v17.16b, #4 // 0 + ext v25.16b, v17.16b, v18.16b, #4 + ext v26.16b, v16.16b, v17.16b, #8 // +1 + ext v27.16b, v17.16b, v18.16b, #8 + mul v2.8h, v22.8h, v6.8h // * 6 + mla v2.8h, v0.8h, v4.8h // * 5 -> a +.if \bpc == 8 + ld1 {v31.8b}, [x1], #8 +.else + ld1 {v31.8h}, [x1], #16 +.endif + add v16.4s, v16.4s, v26.4s // -1, +1 + add v17.4s, v17.4s, v27.4s +.if \bpc == 8 + uxtl v31.8h, v31.8b +.endif + // This is, surprisingly, faster than other variants where the + // mul+mla pairs are further apart, on Cortex A53. + mul v24.4s, v24.4s, v7.4s // * 6 + mla v24.4s, v16.4s, v5.4s // * 5 -> b + mul v25.4s, v25.4s, v7.4s // * 6 + mla v25.4s, v17.4s, v5.4s // * 5 -> b + + umlal v24.4s, v2.4h, v31.4h // b + a * src + umlal2 v25.4s, v2.8h, v31.8h + mov v0.16b, v1.16b + rshrn v24.4h, v24.4s, #8 + rshrn2 v24.8h, v25.4s, #8 + mov v16.16b, v18.16b + st1 {v24.8h}, [x0], #16 + + b.le 5f + ld1 {v1.8h}, [x4], #16 + ld1 {v17.4s, v18.4s}, [x3], #32 + b 4b + +5: + subs x6, x6, #1 + b.le 0f + mov x5, x11 + add x0, x0, x10, lsl #1 + add x1, x1, x2 + mov x3, x13 // Rewind x3/x4 to where they started + mov x4, x14 + b 1b +0: + ret +endfunc + +// void dav1d_sgr_weighted1_Xbpc_neon(pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int w, const int h, +// const int wt, const int bitdepth_max); +function sgr_weighted1_\bpc\()bpc_neon, export=1 +.if \bpc == 16 + ldr w8, [sp] +.endif + dup v31.8h, w7 + cmp x6, #2 +.if \bpc == 16 + dup v30.8h, w8 +.endif + add x9, x0, x1 + add x10, x2, x3 + add x11, x4, #2*FILTER_OUT_STRIDE + mov x7, #(4*FILTER_OUT_STRIDE) + lsl x1, x1, #1 + lsl x3, x3, #1 + add x8, x5, #7 + bic x8, x8, #7 // Aligned width +.if \bpc == 8 + sub x1, x1, x8 + sub x3, x3, x8 +.else + sub x1, x1, x8, lsl #1 + sub x3, x3, x8, lsl #1 +.endif + sub x7, x7, x8, lsl #1 + mov x8, x5 + b.lt 2f +1: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 + ld1 {v4.8b}, [x10], #8 +.else + ld1 {v0.8h}, [x2], #16 + ld1 {v4.8h}, [x10], #16 +.endif + ld1 {v1.8h}, [x4], #16 + ld1 {v5.8h}, [x11], #16 + subs x5, x5, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u + ushll v4.8h, v4.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u + shl v4.8h, v4.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + sub v5.8h, v5.8h, v4.8h // t1 - u + ushll v2.4s, v0.4h, #7 // u << 7 + ushll2 v3.4s, v0.8h, #7 // u << 7 + ushll v6.4s, v4.4h, #7 // u << 7 + ushll2 v7.4s, v4.8h, #7 // u << 7 + smlal v2.4s, v1.4h, v31.4h // v + smlal2 v3.4s, v1.8h, v31.8h // v + smlal v6.4s, v5.4h, v31.4h // v + smlal2 v7.4s, v5.8h, v31.8h // v +.if \bpc == 8 + rshrn v2.4h, v2.4s, #11 + rshrn2 v2.8h, v3.4s, #11 + rshrn v6.4h, v6.4s, #11 + rshrn2 v6.8h, v7.4s, #11 + sqxtun v2.8b, v2.8h + sqxtun v6.8b, v6.8h + st1 {v2.8b}, [x0], #8 + st1 {v6.8b}, [x9], #8 +.else + sqrshrun v2.4h, v2.4s, #11 + sqrshrun2 v2.8h, v3.4s, #11 + sqrshrun v6.4h, v6.4s, #11 + sqrshrun2 v6.8h, v7.4s, #11 + umin v2.8h, v2.8h, v30.8h + umin v6.8h, v6.8h, v30.8h + st1 {v2.8h}, [x0], #16 + st1 {v6.8h}, [x9], #16 +.endif + b.gt 1b + + sub x6, x6, #2 + cmp x6, #1 + b.lt 0f + mov x5, x8 + add x0, x0, x1 + add x9, x9, x1 + add x2, x2, x3 + add x10, x10, x3 + add x4, x4, x7 + add x11, x11, x7 + b.eq 2f + b 1b + +2: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 +.else + ld1 {v0.8h}, [x2], #16 +.endif + ld1 {v1.8h}, [x4], #16 + subs x5, x5, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + ushll v2.4s, v0.4h, #7 // u << 7 + ushll2 v3.4s, v0.8h, #7 // u << 7 + smlal v2.4s, v1.4h, v31.4h // v + smlal2 v3.4s, v1.8h, v31.8h // v +.if \bpc == 8 + rshrn v2.4h, v2.4s, #11 + rshrn2 v2.8h, v3.4s, #11 + sqxtun v2.8b, v2.8h + st1 {v2.8b}, [x0], #8 +.else + sqrshrun v2.4h, v2.4s, #11 + sqrshrun2 v2.8h, v3.4s, #11 + umin v2.8h, v2.8h, v30.8h + st1 {v2.8h}, [x0], #16 +.endif + b.gt 2b +0: + ret +endfunc + +// void dav1d_sgr_weighted2_Xbpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int16_t *t2, +// const int w, const int h, +// const int16_t wt[2]); +function sgr_weighted2_\bpc\()bpc_neon, export=1 +.if \bpc == 8 + ldr x8, [sp] +.else + ldp x8, x9, [sp] +.endif + cmp x7, #2 + add x10, x0, x1 + add x11, x2, x3 + add x12, x4, #2*FILTER_OUT_STRIDE + add x13, x5, #2*FILTER_OUT_STRIDE + ld2r {v30.8h, v31.8h}, [x8] // wt[0], wt[1] +.if \bpc == 16 + dup v29.8h, w9 +.endif + mov x8, #4*FILTER_OUT_STRIDE + lsl x1, x1, #1 + lsl x3, x3, #1 + add x9, x6, #7 + bic x9, x9, #7 // Aligned width +.if \bpc == 8 + sub x1, x1, x9 + sub x3, x3, x9 +.else + sub x1, x1, x9, lsl #1 + sub x3, x3, x9, lsl #1 +.endif + sub x8, x8, x9, lsl #1 + mov x9, x6 + b.lt 2f +1: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 + ld1 {v16.8b}, [x11], #8 +.else + ld1 {v0.8h}, [x2], #16 + ld1 {v16.8h}, [x11], #16 +.endif + ld1 {v1.8h}, [x4], #16 + ld1 {v17.8h}, [x12], #16 + ld1 {v2.8h}, [x5], #16 + ld1 {v18.8h}, [x13], #16 + subs x6, x6, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u + ushll v16.8h, v16.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u + shl v16.8h, v16.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + sub v2.8h, v2.8h, v0.8h // t2 - u + sub v17.8h, v17.8h, v16.8h // t1 - u + sub v18.8h, v18.8h, v16.8h // t2 - u + ushll v3.4s, v0.4h, #7 // u << 7 + ushll2 v4.4s, v0.8h, #7 // u << 7 + ushll v19.4s, v16.4h, #7 // u << 7 + ushll2 v20.4s, v16.8h, #7 // u << 7 + smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) + smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) + smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) + smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) + smlal v19.4s, v17.4h, v30.4h // wt[0] * (t1 - u) + smlal v19.4s, v18.4h, v31.4h // wt[1] * (t2 - u) + smlal2 v20.4s, v17.8h, v30.8h // wt[0] * (t1 - u) + smlal2 v20.4s, v18.8h, v31.8h // wt[1] * (t2 - u) +.if \bpc == 8 + rshrn v3.4h, v3.4s, #11 + rshrn2 v3.8h, v4.4s, #11 + rshrn v19.4h, v19.4s, #11 + rshrn2 v19.8h, v20.4s, #11 + sqxtun v3.8b, v3.8h + sqxtun v19.8b, v19.8h + st1 {v3.8b}, [x0], #8 + st1 {v19.8b}, [x10], #8 +.else + sqrshrun v3.4h, v3.4s, #11 + sqrshrun2 v3.8h, v4.4s, #11 + sqrshrun v19.4h, v19.4s, #11 + sqrshrun2 v19.8h, v20.4s, #11 + umin v3.8h, v3.8h, v29.8h + umin v19.8h, v19.8h, v29.8h + st1 {v3.8h}, [x0], #16 + st1 {v19.8h}, [x10], #16 +.endif + b.gt 1b + + subs x7, x7, #2 + cmp x7, #1 + b.lt 0f + mov x6, x9 + add x0, x0, x1 + add x10, x10, x1 + add x2, x2, x3 + add x11, x11, x3 + add x4, x4, x8 + add x12, x12, x8 + add x5, x5, x8 + add x13, x13, x8 + b.eq 2f + b 1b + +2: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 +.else + ld1 {v0.8h}, [x2], #16 +.endif + ld1 {v1.8h}, [x4], #16 + ld1 {v2.8h}, [x5], #16 + subs x6, x6, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + sub v2.8h, v2.8h, v0.8h // t2 - u + ushll v3.4s, v0.4h, #7 // u << 7 + ushll2 v4.4s, v0.8h, #7 // u << 7 + smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) + smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) + smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) + smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) +.if \bpc == 8 + rshrn v3.4h, v3.4s, #11 + rshrn2 v3.8h, v4.4s, #11 + sqxtun v3.8b, v3.8h + st1 {v3.8b}, [x0], #8 +.else + sqrshrun v3.4h, v3.4s, #11 + sqrshrun2 v3.8h, v4.4s, #11 + umin v3.8h, v3.8h, v29.8h + st1 {v3.8h}, [x0], #16 +.endif + b.gt 1b +0: + ret +endfunc +.endm diff --git a/third_party/dav1d/src/arm/64/mc.S b/third_party/dav1d/src/arm/64/mc.S --- a/third_party/dav1d/src/arm/64/mc.S +++ b/third_party/dav1d/src/arm/64/mc.S @@ -27,36 +27,20 @@ */ #include "src/arm/asm.S" #include "util.S" -.macro avg dst, t0, t1 - ld1 {\t0\().8h}, [x2], 16 - ld1 {\t1\().8h}, [x3], 16 - add \t0\().8h, \t0\().8h, \t1\().8h - sqrshrun \dst\().8b, \t0\().8h, #5 -.endm - -.macro avg16 dst, t0, t1, t2, t3 +.macro avg dst, t0, t1, t2, t3 ld1 {\t0\().8h,\t1\().8h}, [x2], 32 ld1 {\t2\().8h,\t3\().8h}, [x3], 32 add \t0\().8h, \t0\().8h, \t2\().8h add \t1\().8h, \t1\().8h, \t3\().8h sqrshrun \dst\().8b, \t0\().8h, #5 sqrshrun2 \dst\().16b, \t1\().8h, #5 .endm -.macro w_avg dst, t0, t1 - ld1 {\t0\().8h}, [x2], 16 - ld1 {\t1\().8h}, [x3], 16 - sub \t0\().8h, \t1\().8h, \t0\().8h - sqdmulh \t0\().8h, \t0\().8h, v30.8h - add \t0\().8h, \t1\().8h, \t0\().8h - sqrshrun \dst\().8b, \t0\().8h, #4 -.endm - -.macro w_avg16 dst, t0, t1, t2, t3 +.macro w_avg dst, t0, t1, t2, t3 ld1 {\t0\().8h,\t1\().8h}, [x2], 32 ld1 {\t2\().8h,\t3\().8h}, [x3], 32 sub \t0\().8h, \t2\().8h, \t0\().8h sub \t1\().8h, \t3\().8h, \t1\().8h sqdmulh \t0\().8h, \t0\().8h, v30.8h @@ -65,23 +49,11 @@ add \t1\().8h, \t3\().8h, \t1\().8h sqrshrun \dst\().8b, \t0\().8h, #4 sqrshrun2 \dst\().16b, \t1\().8h, #4 .endm -.macro mask dst, t0, t1 - ld1 {v30.8b}, [x6], 8 - ld1 {\t0\().8h}, [x2], 16 - mul v30.8b, v30.8b, v31.8b - ld1 {\t1\().8h}, [x3], 16 - shll v30.8h, v30.8b, #8 - sub \t0\().8h, \t1\().8h, \t0\().8h - sqdmulh \t0\().8h, \t0\().8h, v30.8h - add \t0\().8h, \t1\().8h, \t0\().8h - sqrshrun \dst\().8b, \t0\().8h, #4 -.endm - -.macro mask16 dst, t0, t1, t2, t3 +.macro mask dst, t0, t1, t2, t3 ld1 {v30.16b}, [x6], 16 ld1 {\t0\().8h,\t1\().8h}, [x2], 32 mul v30.16b, v30.16b, v31.16b ld1 {\t2\().8h,\t3\().8h}, [x3], 32 shll v28.8h, v30.8b, #8 @@ -107,127 +79,122 @@ .ifc \type, mask movi v31.16b, #256-2 .endif adr x7, L(\type\()_tbl) sub w4, w4, #24 - \type v4, v0, v1 ldrh w4, [x7, x4, lsl #1] - \type v5, v2, v3 + \type v4, v0, v1, v2, v3 sub x7, x7, w4, uxtw br x7 +40: + add x7, x0, x1 + lsl x1, x1, #1 4: cmp w5, #4 st1 {v4.s}[0], [x0], x1 - st1 {v4.s}[1], [x0], x1 - st1 {v5.s}[0], [x0], x1 - st1 {v5.s}[1], [x0], x1 + st1 {v4.s}[1], [x7], x1 + st1 {v4.s}[2], [x0], x1 + st1 {v4.s}[3], [x7], x1 b.eq 0f - \type v6, v0, v1 - \type v7, v2, v3 + \type v5, v0, v1, v2, v3 cmp w5, #8 - st1 {v6.s}[0], [x0], x1 - st1 {v6.s}[1], [x0], x1 - st1 {v7.s}[0], [x0], x1 - st1 {v7.s}[1], [x0], x1 + st1 {v5.s}[0], [x0], x1 + st1 {v5.s}[1], [x7], x1 + st1 {v5.s}[2], [x0], x1 + st1 {v5.s}[3], [x7], x1 b.eq 0f - \type v4, v0, v1 - \type v5, v2, v3 + \type v4, v0, v1, v2, v3 st1 {v4.s}[0], [x0], x1 - st1 {v4.s}[1], [x0], x1 - \type v6, v0, v1 + st1 {v4.s}[1], [x7], x1 + \type v5, v0, v1, v2, v3 + st1 {v4.s}[2], [x0], x1 + st1 {v4.s}[3], [x7], x1 st1 {v5.s}[0], [x0], x1 - st1 {v5.s}[1], [x0], x1 - \type v7, v2, v3 - st1 {v6.s}[0], [x0], x1 - st1 {v6.s}[1], [x0], x1 - st1 {v7.s}[0], [x0], x1 - st1 {v7.s}[1], [x0], x1 + st1 {v5.s}[1], [x7], x1 + st1 {v5.s}[2], [x0], x1 + st1 {v5.s}[3], [x7], x1 ret +80: + add x7, x0, x1 + lsl x1, x1, #1 8: - st1 {v4.8b}, [x0], x1 - \type v6, v0, v1 - st1 {v5.8b}, [x0], x1 - \type v7, v0, v1 - st1 {v6.8b}, [x0], x1 + st1 {v4.d}[0], [x0], x1 + \type v5, v0, v1, v2, v3 + st1 {v4.d}[1], [x7], x1 + st1 {v5.d}[0], [x0], x1 subs w5, w5, #4 - st1 {v7.8b}, [x0], x1 + st1 {v5.d}[1], [x7], x1 b.le 0f - \type v4, v0, v1 - \type v5, v2, v3 + \type v4, v0, v1, v2, v3 b 8b -160: - trn1 v4.2d, v4.2d, v5.2d 16: - \type\()16 v5, v0, v1, v2, v3 + \type v5, v0, v1, v2, v3 st1 {v4.16b}, [x0], x1 - \type\()16 v6, v0, v1, v2, v3 + \type v6, v0, v1, v2, v3 st1 {v5.16b}, [x0], x1 - \type\()16 v7, v0, v1, v2, v3 + \type v7, v0, v1, v2, v3 st1 {v6.16b}, [x0], x1 subs w5, w5, #4 st1 {v7.16b}, [x0], x1 b.le 0f - \type\()16 v4, v0, v1, v2, v3 + \type v4, v0, v1, v2, v3 b 16b 320: - trn1 v4.2d, v4.2d, v5.2d add x7, x0, x1 lsl x1, x1, #1 32: - \type\()16 v5, v0, v1, v2, v3 - \type\()16 v6, v0, v1, v2, v3 + \type v5, v0, v1, v2, v3 + \type v6, v0, v1, v2, v3 st1 {v4.16b,v5.16b}, [x0], x1 - \type\()16 v7, v0, v1, v2, v3 + \type v7, v0, v1, v2, v3 subs w5, w5, #2 st1 {v6.16b,v7.16b}, [x7], x1 b.le 0f - \type\()16 v4, v0, v1, v2, v3 + \type v4, v0, v1, v2, v3 b 32b 640: - trn1 v4.2d, v4.2d, v5.2d add x7, x0, x1 lsl x1, x1, #1 64: - \type\()16 v5, v0, v1, v2, v3 - \type\()16 v6, v0, v1, v2, v3 - \type\()16 v7, v0, v1, v2, v3 - \type\()16 v16, v0, v1, v2, v3 - \type\()16 v17, v0, v1, v2, v3 + \type v5, v0, v1, v2, v3 + \type v6, v0, v1, v2, v3 + \type v7, v0, v1, v2, v3 + \type v16, v0, v1, v2, v3 + \type v17, v0, v1, v2, v3 st1 {v4.16b,v5.16b,v6.16b,v7.16b}, [x0], x1 - \type\()16 v18, v0, v1, v2, v3 - \type\()16 v19, v0, v1, v2, v3 + \type v18, v0, v1, v2, v3 + \type v19, v0, v1, v2, v3 subs w5, w5, #2 st1 {v16.16b,v17.16b,v18.16b,v19.16b}, [x7], x1 b.le 0f - \type\()16 v4, v0, v1, v2, v3 + \type v4, v0, v1, v2, v3 b 64b 1280: - trn1 v4.2d, v4.2d, v5.2d add x7, x0, #64 128: - \type\()16 v5, v0, v1, v2, v3 - \type\()16 v6, v0, v1, v2, v3 - \type\()16 v7, v0, v1, v2, v3 - \type\()16 v16, v0, v1, v2, v3 - \type\()16 v17, v0, v1, v2, v3 + \type v5, v0, v1, v2, v3 + \type v6, v0, v1, v2, v3 + \type v7, v0, v1, v2, v3 + \type v16, v0, v1, v2, v3 + \type v17, v0, v1, v2, v3 st1 {v4.16b,v5.16b,v6.16b,v7.16b}, [x0], x1 - \type\()16 v18, v0, v1, v2, v3 - \type\()16 v19, v0, v1, v2, v3 + \type v18, v0, v1, v2, v3 + \type v19, v0, v1, v2, v3 subs w5, w5, #1 st1 {v16.16b,v17.16b,v18.16b,v19.16b}, [x7], x1 b.le 0f - \type\()16 v4, v0, v1, v2, v3 + \type v4, v0, v1, v2, v3 b 128b 0: ret L(\type\()_tbl): .hword L(\type\()_tbl) - 1280b .hword L(\type\()_tbl) - 640b .hword L(\type\()_tbl) - 320b - .hword L(\type\()_tbl) - 160b - .hword L(\type\()_tbl) - 8b - .hword L(\type\()_tbl) - 4b + .hword L(\type\()_tbl) - 16b + .hword L(\type\()_tbl) - 80b + .hword L(\type\()_tbl) - 40b endfunc .endm bidir_fn avg bidir_fn w_avg @@ -462,14 +429,14 @@ sub w3, w3, #26 ldrh w3, [x6, x3, lsl #1] sub x6, x6, w3, uxtw movi v4.16b, #64 add x8, x0, x1 - lsl w1, w1, #1 + lsl x1, x1, #1 br x6 4: - ld1 {v2.d}[0], [x5], #8 + ld1 {v2.8b}, [x5], #8 ld1 {v1.d}[0], [x2], #8 ld1 {v0.s}[0], [x0] subs w4, w4, #2 ld1 {v0.s}[1], [x8] sub v3.8b, v4.8b, v2.8b @@ -479,12 +446,12 @@ st1 {v6.s}[0], [x0], x1 st1 {v6.s}[1], [x8], x1 b.gt 4b ret 8: - ld1 {v2.2d}, [x5], #16 - ld1 {v1.2d}, [x2], #16 + ld1 {v2.16b}, [x5], #16 + ld1 {v1.16b}, [x2], #16 ld1 {v0.d}[0], [x0] ld1 {v0.d}[1], [x8] sub v3.16b, v4.16b, v2.16b subs w4, w4, #2 umull v5.8h, v1.8b, v2.8b @@ -496,17 +463,17 @@ st1 {v7.d}[0], [x0], x1 st1 {v7.d}[1], [x8], x1 b.gt 8b ret 16: - ld1 {v1.2d, v2.2d}, [x5], #32 - ld1 {v5.2d, v6.2d}, [x2], #32 - ld1 {v0.2d}, [x0] + ld1 {v1.16b, v2.16b}, [x5], #32 + ld1 {v5.16b, v6.16b}, [x2], #32 + ld1 {v0.16b}, [x0] subs w4, w4, #2 sub v7.16b, v4.16b, v1.16b sub v20.16b, v4.16b, v2.16b - ld1 {v3.2d}, [x8] + ld1 {v3.16b}, [x8] umull v16.8h, v5.8b, v1.8b umlal v16.8h, v0.8b, v7.8b umull2 v17.8h, v5.16b, v1.16b umlal2 v17.8h, v0.16b, v7.16b umull v21.8h, v6.8b, v2.8b @@ -515,20 +482,20 @@ umlal2 v22.8h, v3.16b, v20.16b rshrn v18.8b, v16.8h, #6 rshrn2 v18.16b, v17.8h, #6 rshrn v19.8b, v21.8h, #6 rshrn2 v19.16b, v22.8h, #6 - st1 {v18.2d}, [x0], x1 - st1 {v19.2d}, [x8], x1 + st1 {v18.16b}, [x0], x1 + st1 {v19.16b}, [x8], x1 b.gt 16b ret 32: - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x5], #64 - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x2], #64 - ld1 {v20.2d, v21.2d}, [x0] + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x5], #64 + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x2], #64 + ld1 {v20.16b, v21.16b}, [x0] subs w4, w4, #2 - ld1 {v22.2d, v23.2d}, [x8] + ld1 {v22.16b, v23.16b}, [x8] sub v5.16b, v4.16b, v0.16b sub v6.16b, v4.16b, v1.16b sub v30.16b, v4.16b, v2.16b sub v31.16b, v4.16b, v3.16b umull v24.8h, v16.8b, v0.8b @@ -553,23 +520,23 @@ rshrn2 v25.16b, v7.8h, #6 rshrn v27.8b, v27.8h, #6 rshrn2 v27.16b, v1.8h, #6 rshrn v28.8b, v29.8h, #6 rshrn2 v28.16b, v21.8h, #6 - st1 {v24.2d, v25.2d}, [x0], x1 - st1 {v27.2d, v28.2d}, [x8], x1 + st1 {v24.16b, v25.16b}, [x0], x1 + st1 {v27.16b, v28.16b}, [x8], x1 b.gt 32b ret L(blend_tbl): .hword L(blend_tbl) - 32b .hword L(blend_tbl) - 16b .hword L(blend_tbl) - 8b .hword L(blend_tbl) - 4b endfunc function blend_h_8bpc_neon, export=1 - adr x6, L(blend_h_tbl) + adr x6, L(blend_h_tbl) movrel x5, X(obmc_masks) add x5, x5, w4, uxtw sub w4, w4, w4, lsr #2 clz w7, w3 movi v4.16b, #64 @@ -594,11 +561,11 @@ st1 {v5.h}[1], [x8], x1 b.gt 2b ret 4: ld2r {v0.8b, v1.8b}, [x5], #2 - ld1 {v2.2s}, [x2], #8 + ld1 {v2.8b}, [x2], #8 subs w4, w4, #2 ext v0.8b, v0.8b, v1.8b, #4 ld1 {v3.s}[0], [x0] sub v5.8b, v4.8b, v0.8b ld1 {v3.s}[1], [x8] @@ -740,32 +707,30 @@ st1 {v5.b}[1], [x8], x1 b.gt 2b ret 40: ld1r {v0.2s}, [x5] + sub x1, x1, #2 sub v1.8b, v4.8b, v0.8b - sub x1, x1, #3 4: ld1 {v2.8b}, [x2], #8 ld1 {v3.s}[0], [x0] ld1 {v3.s}[1], [x8] subs w4, w4, #2 umull v5.8h, v2.8b, v0.8b umlal v5.8h, v3.8b, v1.8b rshrn v5.8b, v5.8h, #6 st1 {v5.h}[0], [x0], #2 st1 {v5.h}[2], [x8], #2 - st1 {v5.b}[2], [x0], #1 - st1 {v5.b}[6], [x8], #1 - add x0, x0, x1 - add x8, x8, x1 + st1 {v5.b}[2], [x0], x1 + st1 {v5.b}[6], [x8], x1 b.gt 4b ret 80: ld1r {v0.2d}, [x5] + sub x1, x1, #4 sub v1.16b, v4.16b, v0.16b - sub x1, x1, #6 8: ld1 {v2.16b}, [x2], #16 ld1 {v3.d}[0], [x0] ld1 {v3.d}[1], [x8] subs w4, w4, #2 @@ -775,20 +740,18 @@ umlal2 v6.8h, v3.16b, v1.16b rshrn v7.8b, v5.8h, #6 rshrn2 v7.16b, v6.8h, #6 st1 {v7.s}[0], [x0], #4 st1 {v7.s}[2], [x8], #4 - st1 {v7.h}[2], [x0], #2 - st1 {v7.h}[6], [x8], #2 - add x0, x0, x1 - add x8, x8, x1 + st1 {v7.h}[2], [x0], x1 + st1 {v7.h}[6], [x8], x1 b.gt 8b ret 160: ld1 {v0.16b}, [x5] + sub x1, x1, #8 sub v2.16b, v4.16b, v0.16b - sub x1, x1, #12 16: ld1 {v5.16b, v6.16b}, [x2], #32 ld1 {v7.16b}, [x0] subs w4, w4, #2 ld1 {v16.16b}, [x8] @@ -804,21 +767,19 @@ rshrn2 v19.16b, v18.8h, #6 rshrn v22.8b, v20.8h, #6 rshrn2 v22.16b, v21.8h, #6 st1 {v19.8b}, [x0], #8 st1 {v22.8b}, [x8], #8 - st1 {v19.s}[2], [x0], #4 - st1 {v22.s}[2], [x8], #4 - add x0, x0, x1 - add x8, x8, x1 + st1 {v19.s}[2], [x0], x1 + st1 {v22.s}[2], [x8], x1 b.gt 16b ret 320: ld1 {v0.16b, v1.16b}, [x5] + sub x1, x1, #16 sub v2.16b, v4.16b, v0.16b - sub v3.16b, v4.16b, v1.16b - sub x1, x1, #24 + sub v3.8b, v4.8b, v1.8b 32: ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x2], #64 ld1 {v5.16b, v6.16b}, [x0] subs w4, w4, #2 ld1 {v20.16b, v21.16b}, [x8] @@ -826,34 +787,26 @@ umlal v22.8h, v5.8b, v2.8b umull2 v23.8h, v16.16b, v0.16b umlal2 v23.8h, v5.16b, v2.16b umull v28.8h, v17.8b, v1.8b umlal v28.8h, v6.8b, v3.8b - umull2 v29.8h, v17.16b, v1.16b - umlal2 v29.8h, v6.16b, v3.16b umull v30.8h, v18.8b, v0.8b umlal v30.8h, v20.8b, v2.8b umull2 v31.8h, v18.16b, v0.16b umlal2 v31.8h, v20.16b, v2.16b umull v25.8h, v19.8b, v1.8b umlal v25.8h, v21.8b, v3.8b - umull2 v26.8h, v19.16b, v1.16b - umlal2 v26.8h, v21.16b, v3.16b rshrn v24.8b, v22.8h, #6 rshrn2 v24.16b, v23.8h, #6 rshrn v28.8b, v28.8h, #6 - rshrn2 v28.16b, v29.8h, #6 rshrn v30.8b, v30.8h, #6 rshrn2 v30.16b, v31.8h, #6 rshrn v27.8b, v25.8h, #6 - rshrn2 v27.16b, v26.8h, #6 st1 {v24.16b}, [x0], #16 st1 {v30.16b}, [x8], #16 - st1 {v28.8b}, [x0], #8 - st1 {v27.8b}, [x8], #8 - add x0, x0, x1 - add x8, x8, x1 + st1 {v28.8b}, [x0], x1 + st1 {v27.8b}, [x8], x1 b.gt 32b ret L(blend_v_tbl): .hword L(blend_v_tbl) - 320b .hword L(blend_v_tbl) - 160b @@ -2104,13 +2057,13 @@ .else st1 {v2.4h}, [\dst], \d_strd st1 {v3.4h}, [\ds2], \d_strd .endif b.le 0f - mov v16.16b, v18.16b - mov v17.16b, v28.16b - mov v18.16b, v29.16b + mov v16.8b, v18.8b + mov v17.8b, v28.8b + mov v18.8b, v29.8b b 4b 480: // 4x8, 4x16, 4x32 hv ld1 {v1.8b}, [\xmy] sub \src, \src, #1 @@ -3134,5 +3087,163 @@ endfunc .endm warp , 11 warp t, 7 + +// void dav1d_emu_edge_8bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_8bpc_neon, export=1 + ldp x8, x9, [sp] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub x12, x3, #1 // ih - 1 + cmp x5, x3 + sub x13, x2, #1 // iw - 1 + csel x12, x12, x5, ge // min(y, ih - 1) + cmp x4, x2 + bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0) + csel x13, x13, x4, ge // min(x, iw - 1) + bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0) + madd x8, x12, x9, x8 // ref += iclip() * stride + add x8, x8, x13 // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add x10, x5, x1 // y + bh + neg x5, x5 // -y + sub x10, x10, x3 // y + bh - ih + sub x12, x1, #1 // bh - 1 + cmp x10, x1 + bic x5, x5, x5, asr #63 // max(-y, 0) + csel x10, x10, x12, lt // min(y + bh - ih, bh-1) + cmp x5, x1 + bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0) + csel x5, x5, x12, lt // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add x11, x4, x0 // x + bw + neg x4, x4 // -x + sub x11, x11, x2 // x + bw - iw + sub x13, x0, #1 // bw - 1 + cmp x11, x0 + bic x4, x4, x4, asr #63 // max(-x, 0) + csel x11, x11, x13, lt // min(x + bw - iw, bw-1) + cmp x4, x0 + bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0) + csel x4, x4, x13, lt // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub x1, x1, x5 // bh - top_ext + madd x6, x5, x7, x6 + sub x2, x0, x4 // bw - left_ext + sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext + sub x2, x2, x11 // center_w = bw - left_ext - right_ext + + mov x14, x6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + ld1r {v0.16b}, [x8] + mov x12, x6 // out = dst + mov x3, x4 +1: + subs x3, x3, #16 + st1 {v0.16b}, [x12], #16 + b.gt 1b +.endif + mov x13, x8 + add x12, x6, x4 // out = dst + left_ext + mov x3, x2 +1: + ld1 {v0.16b, v1.16b}, [x13], #32 + subs x3, x3, #32 + st1 {v0.16b, v1.16b}, [x12], #32 + b.gt 1b +.if \need_right + add x3, x8, x2 // in + center_w + sub x3, x3, #1 // in + center_w - 1 + add x12, x6, x4 // dst + left_ext + ld1r {v0.16b}, [x3] + add x12, x12, x2 // out = dst + left_ext + center_w + mov x3, x11 +1: + subs x3, x3, #16 + st1 {v0.16b}, [x12], #16 + b.gt 1b +.endif + + subs x1, x1, #1 // center_h-- + add x6, x6, x7 + add x8, x8, x9 + b.gt 0b +.endm + + cbz x4, 2f + // need_left + cbz x11, 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cbz x11, 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + + cbz x10, 3f + // need_bottom + sub x8, x6, x7 // ref = dst - stride + mov x4, x0 +1: + ld1 {v0.16b, v1.16b}, [x8], #32 + mov x3, x10 +2: + subs x3, x3, #1 + st1 {v0.16b, v1.16b}, [x6], x7 + b.gt 2b + msub x6, x7, x10, x6 // dst -= bottom_ext * stride + subs x4, x4, #32 // bw -= 32 + add x6, x6, #32 // dst += 32 + b.gt 1b + +3: + cbz x5, 3f + // need_top + msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride +1: + ld1 {v0.16b, v1.16b}, [x14], #32 + mov x3, x5 +2: + subs x3, x3, #1 + st1 {v0.16b, v1.16b}, [x6], x7 + b.gt 2b + msub x6, x7, x5, x6 // dst -= top_ext * stride + subs x0, x0, #32 // bw -= 32 + add x6, x6, #32 // dst += 32 + b.gt 1b + +3: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/mc16.S b/third_party/dav1d/src/arm/64/mc16.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/mc16.S @@ -0,0 +1,3569 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define PREP_BIAS 8192 + +.macro avg d0, d1, t0, t1, t2, t3 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + sqadd \t0\().8h, \t0\().8h, \t2\().8h + sqadd \t1\().8h, \t1\().8h, \t3\().8h + smax \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + smax \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + sqsub \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + sqsub \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + sshl \d0\().8h, \t0\().8h, v29.8h // -(intermediate_bits+1) + sshl \d1\().8h, \t1\().8h, v29.8h // -(intermediate_bits+1) +.endm + +.macro w_avg d0, d1, t0, t1, t2, t3 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + // This difference requires a 17 bit range, and all bits are + // significant for the following multiplication. + ssubl \d0\().4s, \t2\().4h, \t0\().4h + ssubl2 \t0\().4s, \t2\().8h, \t0\().8h + ssubl \d1\().4s, \t3\().4h, \t1\().4h + ssubl2 \t1\().4s, \t3\().8h, \t1\().8h + mul \d0\().4s, \d0\().4s, v27.4s + mul \t0\().4s, \t0\().4s, v27.4s + mul \d1\().4s, \d1\().4s, v27.4s + mul \t1\().4s, \t1\().4s, v27.4s + sshr \d0\().4s, \d0\().4s, #4 + sshr \t0\().4s, \t0\().4s, #4 + sshr \d1\().4s, \d1\().4s, #4 + sshr \t1\().4s, \t1\().4s, #4 + saddw \d0\().4s, \d0\().4s, \t2\().4h + saddw2 \t0\().4s, \t0\().4s, \t2\().8h + saddw \d1\().4s, \d1\().4s, \t3\().4h + saddw2 \t1\().4s, \t1\().4s, \t3\().8h + xtn \d0\().4h, \d0\().4s + xtn2 \d0\().8h, \t0\().4s + xtn \d1\().4h, \d1\().4s + xtn2 \d1\().8h, \t1\().4s + srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits + srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits + add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits + add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits + smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max + smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max + smax \d0\().8h, \d0\().8h, v30.8h // 0 + smax \d1\().8h, \d1\().8h, v30.8h // 0 +.endm + +.macro mask d0, d1, t0, t1, t2, t3 + ld1 {v27.16b}, [x6], 16 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + neg v27.16b, v27.16b + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + sxtl v26.8h, v27.8b + sxtl2 v27.8h, v27.16b + sxtl v24.4s, v26.4h + sxtl2 v25.4s, v26.8h + sxtl v26.4s, v27.4h + sxtl2 v27.4s, v27.8h + ssubl \d0\().4s, \t2\().4h, \t0\().4h + ssubl2 \t0\().4s, \t2\().8h, \t0\().8h + ssubl \d1\().4s, \t3\().4h, \t1\().4h + ssubl2 \t1\().4s, \t3\().8h, \t1\().8h + mul \d0\().4s, \d0\().4s, v24.4s + mul \t0\().4s, \t0\().4s, v25.4s + mul \d1\().4s, \d1\().4s, v26.4s + mul \t1\().4s, \t1\().4s, v27.4s + sshr \d0\().4s, \d0\().4s, #6 + sshr \t0\().4s, \t0\().4s, #6 + sshr \d1\().4s, \d1\().4s, #6 + sshr \t1\().4s, \t1\().4s, #6 + saddw \d0\().4s, \d0\().4s, \t2\().4h + saddw2 \t0\().4s, \t0\().4s, \t2\().8h + saddw \d1\().4s, \d1\().4s, \t3\().4h + saddw2 \t1\().4s, \t1\().4s, \t3\().8h + xtn \d0\().4h, \d0\().4s + xtn2 \d0\().8h, \t0\().4s + xtn \d1\().4h, \d1\().4s + xtn2 \d1\().8h, \t1\().4s + srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits + srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits + add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits + add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits + smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max + smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max + smax \d0\().8h, \d0\().8h, v30.8h // 0 + smax \d1\().8h, \d1\().8h, v30.8h // 0 +.endm + +.macro bidir_fn type, bdmax +function \type\()_16bpc_neon, export=1 + clz w4, w4 +.ifnc \type, avg + dup v31.8h, \bdmax // bitdepth_max + movi v30.8h, #0 +.endif + clz w7, \bdmax + sub w7, w7, #18 // intermediate_bits = clz(bitdepth_max) - 18 +.ifc \type, avg + mov w9, #1 + mov w8, #-2*PREP_BIAS + lsl w9, w9, w7 // 1 << intermediate_bits + add w7, w7, #1 + sub w8, w8, w9 // -2*PREP_BIAS - 1 << intermediate_bits + neg w7, w7 // -(intermediate_bits+1) + dup v28.8h, w8 // -2*PREP_BIAS - 1 << intermediate_bits + dup v29.8h, w7 // -(intermediate_bits+1) +.else + mov w8, #PREP_BIAS + lsr w8, w8, w7 // PREP_BIAS >> intermediate_bits + neg w7, w7 // -intermediate_bits + dup v28.8h, w8 // PREP_BIAS >> intermediate_bits + dup v29.8h, w7 // -intermediate_bits +.endif +.ifc \type, w_avg + dup v27.4s, w6 + neg v27.4s, v27.4s +.endif + adr x7, L(\type\()_tbl) + sub w4, w4, #24 + \type v4, v5, v0, v1, v2, v3 + ldrh w4, [x7, x4, lsl #1] + sub x7, x7, w4, uxtw + br x7 +40: + add x7, x0, x1 + lsl x1, x1, #1 +4: + subs w5, w5, #4 + st1 {v4.d}[0], [x0], x1 + st1 {v4.d}[1], [x7], x1 + st1 {v5.d}[0], [x0], x1 + st1 {v5.d}[1], [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 4b +80: + add x7, x0, x1 + lsl x1, x1, #1 +8: + st1 {v4.8h}, [x0], x1 + subs w5, w5, #2 + st1 {v5.8h}, [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 8b +16: + \type v6, v7, v0, v1, v2, v3 + st1 {v4.8h, v5.8h}, [x0], x1 + subs w5, w5, #2 + st1 {v6.8h, v7.8h}, [x0], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 16b +32: + \type v6, v7, v0, v1, v2, v3 + subs w5, w5, #1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 32b +640: + add x7, x0, #64 +64: + \type v6, v7, v0, v1, v2, v3 + \type v16, v17, v0, v1, v2, v3 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + \type v18, v19, v0, v1, v2, v3 + subs w5, w5, #1 + st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 64b +1280: + add x7, x0, #64 + mov x8, #128 + sub x1, x1, #128 +128: + \type v6, v7, v0, v1, v2, v3 + \type v16, v17, v0, v1, v2, v3 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x8 + \type v18, v19, v0, v1, v2, v3 + st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x8 + \type v4, v5, v0, v1, v2, v3 + \type v6, v7, v0, v1, v2, v3 + \type v16, v17, v0, v1, v2, v3 + subs w5, w5, #1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + \type v18, v19, v0, v1, v2, v3 + st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 128b +0: + ret +L(\type\()_tbl): + .hword L(\type\()_tbl) - 1280b + .hword L(\type\()_tbl) - 640b + .hword L(\type\()_tbl) - 32b + .hword L(\type\()_tbl) - 16b + .hword L(\type\()_tbl) - 80b + .hword L(\type\()_tbl) - 40b +endfunc +.endm + +bidir_fn avg, w6 +bidir_fn w_avg, w7 +bidir_fn mask, w7 + + +.macro w_mask_fn type +function w_mask_\type\()_16bpc_neon, export=1 + ldr w8, [sp] + clz w9, w4 + adr x10, L(w_mask_\type\()_tbl) + dup v31.8h, w8 // bitdepth_max + sub w9, w9, #24 + clz w8, w8 // clz(bitdepth_max) + ldrh w9, [x10, x9, lsl #1] + sub x10, x10, w9, uxtw + sub w8, w8, #12 // sh = intermediate_bits + 6 = clz(bitdepth_max) - 12 + mov w9, #PREP_BIAS*64 + neg w8, w8 // -sh + mov w11, #27615 // (64 + 1 - 38)<> mask_sh + ushr v21.8h, v21.8h, #10 + add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 + add v5.4s, v5.4s, v30.4s + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + uxtl v22.4s, v20.4h + uxtl2 v23.4s, v20.8h + uxtl v24.4s, v21.4h + uxtl2 v25.4s, v21.8h + mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m) + mla v5.4s, v17.4s, v23.4s + mla v6.4s, v18.4s, v24.4s + mla v7.4s, v19.4s, v25.4s + srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v5.4s, v5.4s, v29.4s + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + sqxtun v4.4h, v4.4s // iclip_pixel + sqxtun2 v4.8h, v5.4s + sqxtun v5.4h, v6.4s + sqxtun2 v5.8h, v7.4s + umin v4.8h, v4.8h, v31.8h // iclip_pixel + umin v5.8h, v5.8h, v31.8h +.if \type == 444 + xtn v20.8b, v20.8h // 64 - m + xtn2 v20.16b, v21.8h + sub v20.16b, v1.16b, v20.16b // m + st1 {v20.16b}, [x6], #16 +.elseif \type == 422 + addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) + xtn v20.8b, v20.8h + uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + st1 {v20.8b}, [x6], #8 +.elseif \type == 420 + trn1 v24.2d, v20.2d, v21.2d + trn2 v25.2d, v20.2d, v21.2d + add v24.8h, v24.8h, v25.8h // (64 - my1) + (64 - my2) (row wise addition) + addp v20.8h, v24.8h, v24.8h // (128 - m) + (128 - n) (column wise addition) + sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n)) + rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + st1 {v20.s}[0], [x6], #4 +.endif + st1 {v4.d}[0], [x0], x1 + st1 {v4.d}[1], [x12], x1 + st1 {v5.d}[0], [x0], x1 + st1 {v5.d}[1], [x12], x1 + b.gt 4b + ret +8: + ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 + ld1 {v6.8h, v7.8h}, [x3], #32 // tmp2 + subs w5, w5, #2 + sabd v20.8h, v4.8h, v6.8h // abs(tmp1 - tmp2) + sabd v21.8h, v5.8h, v7.8h + ssubl v16.4s, v6.4h, v4.4h // tmp2 - tmp1 (requires 17 bit) + ssubl2 v17.4s, v6.8h, v4.8h + ssubl v18.4s, v7.4h, v5.4h + ssubl2 v19.4s, v7.8h, v5.8h + uqsub v20.8h, v0.8h, v20.8h // 27615 - abs() + uqsub v21.8h, v0.8h, v21.8h + sshll2 v7.4s, v5.8h, #6 // tmp1 << 6 + sshll v6.4s, v5.4h, #6 + sshll2 v5.4s, v4.8h, #6 + sshll v4.4s, v4.4h, #6 + ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh + ushr v21.8h, v21.8h, #10 + add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 + add v5.4s, v5.4s, v30.4s + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + uxtl v22.4s, v20.4h + uxtl2 v23.4s, v20.8h + uxtl v24.4s, v21.4h + uxtl2 v25.4s, v21.8h + mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m) + mla v5.4s, v17.4s, v23.4s + mla v6.4s, v18.4s, v24.4s + mla v7.4s, v19.4s, v25.4s + srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v5.4s, v5.4s, v29.4s + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + sqxtun v4.4h, v4.4s // iclip_pixel + sqxtun2 v4.8h, v5.4s + sqxtun v5.4h, v6.4s + sqxtun2 v5.8h, v7.4s + umin v4.8h, v4.8h, v31.8h // iclip_pixel + umin v5.8h, v5.8h, v31.8h +.if \type == 444 + xtn v20.8b, v20.8h // 64 - m + xtn2 v20.16b, v21.8h + sub v20.16b, v1.16b, v20.16b // m + st1 {v20.16b}, [x6], #16 +.elseif \type == 422 + addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) + xtn v20.8b, v20.8h + uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + st1 {v20.8b}, [x6], #8 +.elseif \type == 420 + add v20.8h, v20.8h, v21.8h // (64 - my1) + (64 - my2) (row wise addition) + addp v20.8h, v20.8h, v20.8h // (128 - m) + (128 - n) (column wise addition) + sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n)) + rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + st1 {v20.s}[0], [x6], #4 +.endif + st1 {v4.8h}, [x0], x1 + st1 {v5.8h}, [x12], x1 + b.gt 8b + ret +1280: +640: +320: +160: + mov w11, w4 + sub x1, x1, w4, uxtw #1 +.if \type == 444 + add x10, x6, w4, uxtw +.elseif \type == 422 + add x10, x6, x11, lsr #1 +.endif + add x9, x3, w4, uxtw #1 + add x7, x2, w4, uxtw #1 +161: + mov w8, w4 +16: + ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 + ld1 {v16.8h, v17.8h}, [x3], #32 // tmp2 + ld1 {v6.8h, v7.8h}, [x7], #32 + ld1 {v18.8h, v19.8h}, [x9], #32 + subs w8, w8, #16 + sabd v20.8h, v4.8h, v16.8h // abs(tmp1 - tmp2) + sabd v21.8h, v5.8h, v17.8h + ssubl v22.4s, v16.4h, v4.4h // tmp2 - tmp1 (requires 17 bit) + ssubl2 v23.4s, v16.8h, v4.8h + ssubl v24.4s, v17.4h, v5.4h + ssubl2 v25.4s, v17.8h, v5.8h + uqsub v20.8h, v0.8h, v20.8h // 27615 - abs() + uqsub v21.8h, v0.8h, v21.8h + sshll2 v27.4s, v5.8h, #6 // tmp1 << 6 + sshll v26.4s, v5.4h, #6 + sshll2 v5.4s, v4.8h, #6 + sshll v4.4s, v4.4h, #6 + ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh + ushr v21.8h, v21.8h, #10 + add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 + add v5.4s, v5.4s, v30.4s + add v26.4s, v26.4s, v30.4s + add v27.4s, v27.4s, v30.4s + uxtl v16.4s, v20.4h + uxtl2 v17.4s, v20.8h + uxtl v28.4s, v21.4h + mla v4.4s, v22.4s, v16.4s // (tmp2-tmp1)*(64-m) + uxtl2 v16.4s, v21.8h + mla v5.4s, v23.4s, v17.4s + mla v26.4s, v24.4s, v28.4s + mla v27.4s, v25.4s, v16.4s + srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v5.4s, v5.4s, v29.4s + srshl v26.4s, v26.4s, v29.4s + srshl v27.4s, v27.4s, v29.4s + sqxtun v4.4h, v4.4s // iclip_pixel + sqxtun2 v4.8h, v5.4s + sqxtun v5.4h, v26.4s + sqxtun2 v5.8h, v27.4s + + // Start of other half + sabd v22.8h, v6.8h, v18.8h // abs(tmp1 - tmp2) + sabd v23.8h, v7.8h, v19.8h + + umin v4.8h, v4.8h, v31.8h // iclip_pixel + umin v5.8h, v5.8h, v31.8h + + ssubl v16.4s, v18.4h, v6.4h // tmp2 - tmp1 (requires 17 bit) + ssubl2 v17.4s, v18.8h, v6.8h + ssubl v18.4s, v19.4h, v7.4h + ssubl2 v19.4s, v19.8h, v7.8h + uqsub v22.8h, v0.8h, v22.8h // 27615 - abs() + uqsub v23.8h, v0.8h, v23.8h + sshll v24.4s, v6.4h, #6 // tmp1 << 6 + sshll2 v25.4s, v6.8h, #6 + sshll v26.4s, v7.4h, #6 + sshll2 v27.4s, v7.8h, #6 + ushr v22.8h, v22.8h, #10 // 64-m = (27615 - abs()) >> mask_sh + ushr v23.8h, v23.8h, #10 + add v24.4s, v24.4s, v30.4s // += PREP_BIAS*64 + add v25.4s, v25.4s, v30.4s + add v26.4s, v26.4s, v30.4s + add v27.4s, v27.4s, v30.4s + uxtl v6.4s, v22.4h + uxtl2 v7.4s, v22.8h + uxtl v28.4s, v23.4h + mla v24.4s, v16.4s, v6.4s // (tmp2-tmp1)*(64-m) + uxtl2 v6.4s, v23.8h + mla v25.4s, v17.4s, v7.4s + mla v26.4s, v18.4s, v28.4s + mla v27.4s, v19.4s, v6.4s + srshl v24.4s, v24.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v25.4s, v25.4s, v29.4s + srshl v26.4s, v26.4s, v29.4s + srshl v27.4s, v27.4s, v29.4s + sqxtun v6.4h, v24.4s // iclip_pixel + sqxtun2 v6.8h, v25.4s + sqxtun v7.4h, v26.4s + sqxtun2 v7.8h, v27.4s + umin v6.8h, v6.8h, v31.8h // iclip_pixel + umin v7.8h, v7.8h, v31.8h +.if \type == 444 + xtn v20.8b, v20.8h // 64 - m + xtn2 v20.16b, v21.8h + xtn v21.8b, v22.8h + xtn2 v21.16b, v23.8h + sub v20.16b, v1.16b, v20.16b // m + sub v21.16b, v1.16b, v21.16b + st1 {v20.16b}, [x6], #16 + st1 {v21.16b}, [x10], #16 +.elseif \type == 422 + addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) + addp v21.8h, v22.8h, v23.8h + xtn v20.8b, v20.8h + xtn v21.8b, v21.8h + uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + uhsub v21.8b, v3.8b, v21.8b + st1 {v20.8b}, [x6], #8 + st1 {v21.8b}, [x10], #8 +.elseif \type == 420 + add v20.8h, v20.8h, v22.8h // (64 - my1) + (64 - my2) (row wise addition) + add v21.8h, v21.8h, v23.8h + addp v20.8h, v20.8h, v21.8h // (128 - m) + (128 - n) (column wise addition) + sub v20.8h, v3.8h, v20.8h // (256 - sign) - ((128 - m) + (128 - n)) + rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + st1 {v20.8b}, [x6], #8 +.endif + st1 {v4.8h, v5.8h}, [x0], #32 + st1 {v6.8h, v7.8h}, [x12], #32 + b.gt 16b + subs w5, w5, #2 + add x2, x2, w4, uxtw #1 + add x3, x3, w4, uxtw #1 + add x7, x7, w4, uxtw #1 + add x9, x9, w4, uxtw #1 +.if \type == 444 + add x6, x6, w4, uxtw + add x10, x10, w4, uxtw +.elseif \type == 422 + add x6, x6, x11, lsr #1 + add x10, x10, x11, lsr #1 +.endif + add x0, x0, x1 + add x12, x12, x1 + b.gt 161b + ret +L(w_mask_\type\()_tbl): + .hword L(w_mask_\type\()_tbl) - 1280b + .hword L(w_mask_\type\()_tbl) - 640b + .hword L(w_mask_\type\()_tbl) - 320b + .hword L(w_mask_\type\()_tbl) - 160b + .hword L(w_mask_\type\()_tbl) - 8b + .hword L(w_mask_\type\()_tbl) - 4b +endfunc +.endm + +w_mask_fn 444 +w_mask_fn 422 +w_mask_fn 420 + + +function blend_16bpc_neon, export=1 + adr x6, L(blend_tbl) + clz w3, w3 + sub w3, w3, #26 + ldrh w3, [x6, x3, lsl #1] + sub x6, x6, w3, uxtw + add x8, x0, x1 + br x6 +40: + lsl x1, x1, #1 +4: + ld1 {v2.8b}, [x5], #8 + ld1 {v1.8h}, [x2], #16 + ld1 {v0.d}[0], [x0] + neg v2.8b, v2.8b // -m + subs w4, w4, #2 + ld1 {v0.d}[1], [x8] + sxtl v2.8h, v2.8b + shl v2.8h, v2.8h, #9 // -m << 9 + sub v1.8h, v0.8h, v1.8h // a - b + sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 + add v0.8h, v0.8h, v1.8h + st1 {v0.d}[0], [x0], x1 + st1 {v0.d}[1], [x8], x1 + b.gt 4b + ret +80: + lsl x1, x1, #1 +8: + ld1 {v4.16b}, [x5], #16 + ld1 {v2.8h, v3.8h}, [x2], #32 + neg v5.16b, v4.16b // -m + ld1 {v0.8h}, [x0] + ld1 {v1.8h}, [x8] + sxtl v4.8h, v5.8b + sxtl2 v5.8h, v5.16b + shl v4.8h, v4.8h, #9 // -m << 9 + shl v5.8h, v5.8h, #9 + sub v2.8h, v0.8h, v2.8h // a - b + sub v3.8h, v1.8h, v3.8h + subs w4, w4, #2 + sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v3.8h, v3.8h, v5.8h + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + st1 {v0.8h}, [x0], x1 + st1 {v1.8h}, [x8], x1 + b.gt 8b + ret +160: + lsl x1, x1, #1 +16: + ld1 {v16.16b, v17.16b}, [x5], #32 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + subs w4, w4, #2 + neg v18.16b, v16.16b // -m + neg v19.16b, v17.16b + ld1 {v0.8h, v1.8h}, [x0] + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b + ld1 {v2.8h, v3.8h}, [x8] + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.8h, v17.8h, #9 + shl v18.8h, v18.8h, #9 + shl v19.8h, v19.8h, #9 + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.8h, v1.8h, v5.8h + sub v6.8h, v2.8h, v6.8h + sub v7.8h, v3.8h, v7.8h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.8h, v5.8h, v17.8h + sqrdmulh v6.8h, v6.8h, v18.8h + sqrdmulh v7.8h, v7.8h, v19.8h + add v0.8h, v0.8h, v4.8h + add v1.8h, v1.8h, v5.8h + add v2.8h, v2.8h, v6.8h + add v3.8h, v3.8h, v7.8h + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v2.8h, v3.8h}, [x8], x1 + b.gt 16b + ret +32: + ld1 {v16.16b, v17.16b}, [x5], #32 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + subs w4, w4, #1 + neg v18.16b, v16.16b // -m + neg v19.16b, v17.16b + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.8h, v17.8h, #9 + shl v18.8h, v18.8h, #9 + shl v19.8h, v19.8h, #9 + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.8h, v1.8h, v5.8h + sub v6.8h, v2.8h, v6.8h + sub v7.8h, v3.8h, v7.8h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.8h, v5.8h, v17.8h + sqrdmulh v6.8h, v6.8h, v18.8h + sqrdmulh v7.8h, v7.8h, v19.8h + add v0.8h, v0.8h, v4.8h + add v1.8h, v1.8h, v5.8h + add v2.8h, v2.8h, v6.8h + add v3.8h, v3.8h, v7.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + b.gt 32b + ret +L(blend_tbl): + .hword L(blend_tbl) - 32b + .hword L(blend_tbl) - 160b + .hword L(blend_tbl) - 80b + .hword L(blend_tbl) - 40b +endfunc + +function blend_h_16bpc_neon, export=1 + adr x6, L(blend_h_tbl) + movrel x5, X(obmc_masks) + add x5, x5, w4, uxtw + sub w4, w4, w4, lsr #2 + clz w7, w3 + add x8, x0, x1 + lsl x1, x1, #1 + sub w7, w7, #24 + ldrh w7, [x6, x7, lsl #1] + sub x6, x6, w7, uxtw + br x6 +2: + ld2r {v2.8b, v3.8b}, [x5], #2 + ld1 {v1.4h}, [x2], #8 + ext v2.8b, v2.8b, v3.8b, #6 + subs w4, w4, #2 + neg v2.8b, v2.8b // -m + ld1 {v0.s}[0], [x0] + ld1 {v0.s}[1], [x8] + sxtl v2.8h, v2.8b + shl v2.4h, v2.4h, #9 // -m << 9 + sub v1.4h, v0.4h, v1.4h // a - b + sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6 + add v0.4h, v0.4h, v1.4h + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[1], [x8], x1 + b.gt 2b + ret +4: + ld2r {v2.8b, v3.8b}, [x5], #2 + ld1 {v1.8h}, [x2], #16 + ext v2.8b, v2.8b, v3.8b, #4 + subs w4, w4, #2 + neg v2.8b, v2.8b // -m + ld1 {v0.d}[0], [x0] + ld1 {v0.d}[1], [x8] + sxtl v2.8h, v2.8b + shl v2.8h, v2.8h, #9 // -m << 9 + sub v1.8h, v0.8h, v1.8h // a - b + sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 + add v0.8h, v0.8h, v1.8h + st1 {v0.d}[0], [x0], x1 + st1 {v0.d}[1], [x8], x1 + b.gt 4b + ret +8: + ld2r {v4.8b, v5.8b}, [x5], #2 + ld1 {v2.8h, v3.8h}, [x2], #32 + neg v4.8b, v4.8b // -m + neg v5.8b, v5.8b + ld1 {v0.8h}, [x0] + subs w4, w4, #2 + sxtl v4.8h, v4.8b + sxtl v5.8h, v5.8b + ld1 {v1.8h}, [x8] + shl v4.8h, v4.8h, #9 // -m << 9 + shl v5.8h, v5.8h, #9 + sub v2.8h, v0.8h, v2.8h // a - b + sub v3.8h, v1.8h, v3.8h + sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v3.8h, v3.8h, v5.8h + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + st1 {v0.8h}, [x0], x1 + st1 {v1.8h}, [x8], x1 + b.gt 8b + ret +16: + ld2r {v16.8b, v17.8b}, [x5], #2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + neg v16.8b, v16.8b // -m + neg v17.8b, v17.8b + ld1 {v0.8h, v1.8h}, [x0] + ld1 {v2.8h, v3.8h}, [x8] + subs w4, w4, #2 + sxtl v16.8h, v16.8b + sxtl v17.8h, v17.8b + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.8h, v17.8h, #9 + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.8h, v1.8h, v5.8h + sub v6.8h, v2.8h, v6.8h + sub v7.8h, v3.8h, v7.8h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.8h, v5.8h, v16.8h + sqrdmulh v6.8h, v6.8h, v17.8h + sqrdmulh v7.8h, v7.8h, v17.8h + add v0.8h, v0.8h, v4.8h + add v1.8h, v1.8h, v5.8h + add v2.8h, v2.8h, v6.8h + add v3.8h, v3.8h, v7.8h + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v2.8h, v3.8h}, [x8], x1 + b.gt 16b + ret +1280: +640: +320: + sub x1, x1, w3, uxtw #1 + add x7, x2, w3, uxtw #1 +321: + ld2r {v24.8b, v25.8b}, [x5], #2 + mov w6, w3 + neg v24.8b, v24.8b // -m + neg v25.8b, v25.8b + sxtl v24.8h, v24.8b + sxtl v25.8h, v25.8b + shl v24.8h, v24.8h, #9 // -m << 9 + shl v25.8h, v25.8h, #9 +32: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64 + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + subs w6, w6, #32 + sub v16.8h, v0.8h, v16.8h // a - b + sub v17.8h, v1.8h, v17.8h + sub v18.8h, v2.8h, v18.8h + sub v19.8h, v3.8h, v19.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8] + sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v17.8h, v17.8h, v24.8h + sqrdmulh v18.8h, v18.8h, v24.8h + sqrdmulh v19.8h, v19.8h, v24.8h + sub v20.8h, v4.8h, v20.8h // a - b + sub v21.8h, v5.8h, v21.8h + sub v22.8h, v6.8h, v22.8h + sub v23.8h, v7.8h, v23.8h + add v0.8h, v0.8h, v16.8h + add v1.8h, v1.8h, v17.8h + add v2.8h, v2.8h, v18.8h + add v3.8h, v3.8h, v19.8h + sqrdmulh v20.8h, v20.8h, v25.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v21.8h, v21.8h, v25.8h + sqrdmulh v22.8h, v22.8h, v25.8h + sqrdmulh v23.8h, v23.8h, v25.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v4.8h, v4.8h, v20.8h + add v5.8h, v5.8h, v21.8h + add v6.8h, v6.8h, v22.8h + add v7.8h, v7.8h, v23.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8], #64 + b.gt 32b + subs w4, w4, #2 + add x0, x0, x1 + add x8, x8, x1 + add x2, x2, w3, uxtw #1 + add x7, x7, w3, uxtw #1 + b.gt 321b + ret +L(blend_h_tbl): + .hword L(blend_h_tbl) - 1280b + .hword L(blend_h_tbl) - 640b + .hword L(blend_h_tbl) - 320b + .hword L(blend_h_tbl) - 16b + .hword L(blend_h_tbl) - 8b + .hword L(blend_h_tbl) - 4b + .hword L(blend_h_tbl) - 2b +endfunc + +function blend_v_16bpc_neon, export=1 + adr x6, L(blend_v_tbl) + movrel x5, X(obmc_masks) + add x5, x5, w3, uxtw + clz w3, w3 + add x8, x0, x1 + lsl x1, x1, #1 + sub w3, w3, #26 + ldrh w3, [x6, x3, lsl #1] + sub x6, x6, w3, uxtw + br x6 +20: + ld1r {v2.8b}, [x5] + neg v2.8b, v2.8b // -m + sxtl v2.8h, v2.8b + shl v2.4h, v2.4h, #9 // -m << 9 +2: + ld1 {v1.s}[0], [x2], #4 + ld1 {v0.h}[0], [x0] + subs w4, w4, #2 + ld1 {v1.h}[1], [x2] + ld1 {v0.h}[1], [x8] + add x2, x2, #4 + sub v1.4h, v0.4h, v1.4h // a - b + sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6 + add v0.4h, v0.4h, v1.4h + st1 {v0.h}[0], [x0], x1 + st1 {v0.h}[1], [x8], x1 + b.gt 2b + ret +40: + ld1r {v2.2s}, [x5] + sub x1, x1, #4 + neg v2.8b, v2.8b // -m + sxtl v2.8h, v2.8b + shl v2.8h, v2.8h, #9 // -m << 9 +4: + ld1 {v1.8h}, [x2], #16 + ld1 {v0.d}[0], [x0] + ld1 {v0.d}[1], [x8] + subs w4, w4, #2 + sub v1.8h, v0.8h, v1.8h // a - b + sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 + add v0.8h, v0.8h, v1.8h + st1 {v0.s}[0], [x0], #4 + st1 {v0.s}[2], [x8], #4 + st1 {v0.h}[2], [x0], x1 + st1 {v0.h}[6], [x8], x1 + b.gt 4b + ret +80: + ld1 {v4.8b}, [x5] + sub x1, x1, #8 + neg v4.8b, v4.8b // -m + sxtl v4.8h, v4.8b + shl v4.8h, v4.8h, #9 // -m << 9 +8: + ld1 {v2.8h, v3.8h}, [x2], #32 + ld1 {v0.8h}, [x0] + ld1 {v1.8h}, [x8] + subs w4, w4, #2 + sub v2.8h, v0.8h, v2.8h // a - b + sub v3.8h, v1.8h, v3.8h + sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v3.8h, v3.8h, v4.8h + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + st1 {v0.d}[0], [x0], #8 + st1 {v1.d}[0], [x8], #8 + st1 {v0.s}[2], [x0], x1 + st1 {v1.s}[2], [x8], x1 + b.gt 8b + ret +160: + ld1 {v16.8b, v17.8b}, [x5] + sub x1, x1, #16 + neg v16.8b, v16.8b // -m + neg v17.8b, v17.8b + sxtl v16.8h, v16.8b + sxtl v17.8h, v17.8b + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.4h, v17.4h, #9 +16: + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + ld1 {v0.8h, v1.8h}, [x0] + subs w4, w4, #2 + ld1 {v2.8h, v3.8h}, [x8] + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.4h, v1.4h, v5.4h + sub v6.8h, v2.8h, v6.8h + sub v7.4h, v3.4h, v7.4h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.4h, v5.4h, v17.4h + sqrdmulh v6.8h, v6.8h, v16.8h + sqrdmulh v7.4h, v7.4h, v17.4h + add v0.8h, v0.8h, v4.8h + add v1.4h, v1.4h, v5.4h + add v2.8h, v2.8h, v6.8h + add v3.4h, v3.4h, v7.4h + st1 {v0.8h}, [x0], #16 + st1 {v2.8h}, [x8], #16 + st1 {v1.4h}, [x0], x1 + st1 {v3.4h}, [x8], x1 + b.gt 16b + ret +320: + ld1 {v24.16b, v25.16b}, [x5] + neg v26.16b, v24.16b // -m + neg v27.8b, v25.8b + sxtl v24.8h, v26.8b + sxtl2 v25.8h, v26.16b + sxtl v26.8h, v27.8b + shl v24.8h, v24.8h, #9 // -m << 9 + shl v25.8h, v25.8h, #9 + shl v26.8h, v26.8h, #9 +32: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64 + ld1 {v0.8h, v1.8h, v2.8h}, [x0] + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2], #64 + ld1 {v4.8h, v5.8h, v6.8h}, [x8] + subs w4, w4, #2 + sub v16.8h, v0.8h, v16.8h // a - b + sub v17.8h, v1.8h, v17.8h + sub v18.8h, v2.8h, v18.8h + sub v20.8h, v4.8h, v20.8h + sub v21.8h, v5.8h, v21.8h + sub v22.8h, v6.8h, v22.8h + sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v17.8h, v17.8h, v25.8h + sqrdmulh v18.8h, v18.8h, v26.8h + sqrdmulh v20.8h, v20.8h, v24.8h + sqrdmulh v21.8h, v21.8h, v25.8h + sqrdmulh v22.8h, v22.8h, v26.8h + add v0.8h, v0.8h, v16.8h + add v1.8h, v1.8h, v17.8h + add v2.8h, v2.8h, v18.8h + add v4.8h, v4.8h, v20.8h + add v5.8h, v5.8h, v21.8h + add v6.8h, v6.8h, v22.8h + st1 {v0.8h, v1.8h, v2.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h}, [x8], x1 + b.gt 32b + ret +L(blend_v_tbl): + .hword L(blend_v_tbl) - 320b + .hword L(blend_v_tbl) - 160b + .hword L(blend_v_tbl) - 80b + .hword L(blend_v_tbl) - 40b + .hword L(blend_v_tbl) - 20b +endfunc + + +// This has got the same signature as the put_8tap functions, +// and assumes that x9 is set to (clz(w)-24). +function put_neon + adr x10, L(put_tbl) + ldrh w9, [x10, x9, lsl #1] + sub x10, x10, w9, uxtw + br x10 + +2: + ld1 {v0.s}[0], [x2], x3 + ld1 {v1.s}[0], [x2], x3 + subs w5, w5, #2 + st1 {v0.s}[0], [x0], x1 + st1 {v1.s}[0], [x0], x1 + b.gt 2b + ret +4: + ld1 {v0.8b}, [x2], x3 + ld1 {v1.8b}, [x2], x3 + subs w5, w5, #2 + st1 {v0.8b}, [x0], x1 + st1 {v1.8b}, [x0], x1 + b.gt 4b + ret +80: + add x8, x0, x1 + lsl x1, x1, #1 + add x9, x2, x3 + lsl x3, x3, #1 +8: + ld1 {v0.16b}, [x2], x3 + ld1 {v1.16b}, [x9], x3 + subs w5, w5, #2 + st1 {v0.16b}, [x0], x1 + st1 {v1.16b}, [x8], x1 + b.gt 8b + ret +16: + ldp x6, x7, [x2] + ldp x8, x9, [x2, #16] + stp x6, x7, [x0] + subs w5, w5, #1 + stp x8, x9, [x0, #16] + add x2, x2, x3 + add x0, x0, x1 + b.gt 16b + ret +32: + ldp x6, x7, [x2] + ldp x8, x9, [x2, #16] + stp x6, x7, [x0] + ldp x10, x11, [x2, #32] + stp x8, x9, [x0, #16] + subs w5, w5, #1 + ldp x12, x13, [x2, #48] + stp x10, x11, [x0, #32] + stp x12, x13, [x0, #48] + add x2, x2, x3 + add x0, x0, x1 + b.gt 32b + ret +64: + ldp q0, q1, [x2] + ldp q2, q3, [x2, #32] + stp q0, q1, [x0] + ldp q4, q5, [x2, #64] + stp q2, q3, [x0, #32] + ldp q6, q7, [x2, #96] + subs w5, w5, #1 + stp q4, q5, [x0, #64] + stp q6, q7, [x0, #96] + add x2, x2, x3 + add x0, x0, x1 + b.gt 64b + ret +128: + ldp q0, q1, [x2] + ldp q2, q3, [x2, #32] + stp q0, q1, [x0] + ldp q4, q5, [x2, #64] + stp q2, q3, [x0, #32] + ldp q6, q7, [x2, #96] + subs w5, w5, #1 + stp q4, q5, [x0, #64] + ldp q16, q17, [x2, #128] + stp q6, q7, [x0, #96] + ldp q18, q19, [x2, #160] + stp q16, q17, [x0, #128] + ldp q20, q21, [x2, #192] + stp q18, q19, [x0, #160] + ldp q22, q23, [x2, #224] + stp q20, q21, [x0, #192] + stp q22, q23, [x0, #224] + add x2, x2, x3 + add x0, x0, x1 + b.gt 128b + ret + +L(put_tbl): + .hword L(put_tbl) - 128b + .hword L(put_tbl) - 64b + .hword L(put_tbl) - 32b + .hword L(put_tbl) - 16b + .hword L(put_tbl) - 80b + .hword L(put_tbl) - 4b + .hword L(put_tbl) - 2b +endfunc + + +// This has got the same signature as the prep_8tap functions, +// and assumes that x9 is set to (clz(w)-24), w7 to intermediate_bits and +// x8 to w*2. +function prep_neon + adr x10, L(prep_tbl) + ldrh w9, [x10, x9, lsl #1] + dup v31.8h, w7 // intermediate_bits + movi v30.8h, #(PREP_BIAS >> 8), lsl #8 + sub x10, x10, w9, uxtw + br x10 + +40: + add x9, x1, x2 + lsl x2, x2, #1 +4: + ld1 {v0.d}[0], [x1], x2 + ld1 {v0.d}[1], [x9], x2 + subs w4, w4, #2 + sshl v0.8h, v0.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + st1 {v0.8h}, [x0], #16 + b.gt 4b + ret +80: + add x9, x1, x2 + lsl x2, x2, #1 +8: + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x9], x2 + subs w4, w4, #2 + sshl v0.8h, v0.8h, v31.8h + sshl v1.8h, v1.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 8b + ret +16: + ldp q0, q1, [x1] + add x1, x1, x2 + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1] + add x1, x1, x2 + subs w4, w4, #2 + sshl v1.8h, v1.8h, v31.8h + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + b.gt 16b + ret +32: + ldp q0, q1, [x1] + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1, #32] + add x1, x1, x2 + sshl v1.8h, v1.8h, v31.8h + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + subs w4, w4, #1 + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + b.gt 32b + ret +64: + ldp q0, q1, [x1] + subs w4, w4, #1 + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1, #32] + sshl v1.8h, v1.8h, v31.8h + ldp q4, q5, [x1, #64] + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + ldp q6, q7, [x1, #96] + add x1, x1, x2 + sshl v4.8h, v4.8h, v31.8h + sshl v5.8h, v5.8h, v31.8h + sshl v6.8h, v6.8h, v31.8h + sshl v7.8h, v7.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + stp q0, q1, [x0] + sub v4.8h, v4.8h, v30.8h + sub v5.8h, v5.8h, v30.8h + stp q2, q3, [x0, #32] + sub v6.8h, v6.8h, v30.8h + sub v7.8h, v7.8h, v30.8h + stp q4, q5, [x0, #64] + stp q6, q7, [x0, #96] + add x0, x0, x8 + b.gt 64b + ret +128: + ldp q0, q1, [x1] + subs w4, w4, #1 + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1, #32] + sshl v1.8h, v1.8h, v31.8h + ldp q4, q5, [x1, #64] + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + ldp q6, q7, [x1, #96] + sshl v4.8h, v4.8h, v31.8h + sshl v5.8h, v5.8h, v31.8h + ldp q16, q17, [x1, #128] + sshl v6.8h, v6.8h, v31.8h + sshl v7.8h, v7.8h, v31.8h + ldp q18, q19, [x1, #160] + sshl v16.8h, v16.8h, v31.8h + sshl v17.8h, v17.8h, v31.8h + ldp q20, q21, [x1, #192] + sshl v18.8h, v18.8h, v31.8h + sshl v19.8h, v19.8h, v31.8h + ldp q22, q23, [x1, #224] + add x1, x1, x2 + sshl v20.8h, v20.8h, v31.8h + sshl v21.8h, v21.8h, v31.8h + sshl v22.8h, v22.8h, v31.8h + sshl v23.8h, v23.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + stp q0, q1, [x0] + sub v4.8h, v4.8h, v30.8h + sub v5.8h, v5.8h, v30.8h + stp q2, q3, [x0, #32] + sub v6.8h, v6.8h, v30.8h + sub v7.8h, v7.8h, v30.8h + stp q4, q5, [x0, #64] + sub v16.8h, v16.8h, v30.8h + sub v17.8h, v17.8h, v30.8h + stp q6, q7, [x0, #96] + sub v18.8h, v18.8h, v30.8h + sub v19.8h, v19.8h, v30.8h + stp q16, q17, [x0, #128] + sub v20.8h, v20.8h, v30.8h + sub v21.8h, v21.8h, v30.8h + stp q18, q19, [x0, #160] + sub v22.8h, v22.8h, v30.8h + sub v23.8h, v23.8h, v30.8h + stp q20, q21, [x0, #192] + stp q22, q23, [x0, #224] + add x0, x0, x8 + b.gt 128b + ret + +L(prep_tbl): + .hword L(prep_tbl) - 128b + .hword L(prep_tbl) - 64b + .hword L(prep_tbl) - 32b + .hword L(prep_tbl) - 16b + .hword L(prep_tbl) - 80b + .hword L(prep_tbl) - 40b +endfunc + + +.macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + ld1 {\d0\wd}[0], [\s0], \strd + ld1 {\d1\wd}[0], [\s1], \strd +.ifnb \d2 + ld1 {\d2\wd}[0], [\s0], \strd + ld1 {\d3\wd}[0], [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd}[0], [\s0], \strd +.endif +.ifnb \d5 + ld1 {\d5\wd}[0], [\s1], \strd +.endif +.ifnb \d6 + ld1 {\d6\wd}[0], [\s0], \strd +.endif +.endm +.macro load_reg s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + ld1 {\d0\wd}, [\s0], \strd + ld1 {\d1\wd}, [\s1], \strd +.ifnb \d2 + ld1 {\d2\wd}, [\s0], \strd + ld1 {\d3\wd}, [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd}, [\s0], \strd +.endif +.ifnb \d5 + ld1 {\d5\wd}, [\s1], \strd +.endif +.ifnb \d6 + ld1 {\d6\wd}, [\s0], \strd +.endif +.endm +.macro load_regpair s0, s1, strd, wd, d0, d1, d2, d3, d4, d5 + ld1 {\d0\wd, \d1\wd}, [\s0], \strd +.ifnb \d2 + ld1 {\d2\wd, \d3\wd}, [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd, \d5\wd}, [\s0], \strd +.endif +.endm +.macro load_s s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_slice \s0, \s1, \strd, .s, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_4h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_reg \s0, \s1, \strd, .4h, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_8h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_reg \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_16h s0, s1, strd, d0, d1, d2, d3, d4, d5 + load_regpair \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5 +.endm +.macro interleave_1 wd, r0, r1, r2, r3, r4 + trn1 \r0\wd, \r0\wd, \r1\wd + trn1 \r1\wd, \r1\wd, \r2\wd +.ifnb \r3 + trn1 \r2\wd, \r2\wd, \r3\wd + trn1 \r3\wd, \r3\wd, \r4\wd +.endif +.endm +.macro interleave_1_s r0, r1, r2, r3, r4 + interleave_1 .2s, \r0, \r1, \r2, \r3, \r4 +.endm +.macro umin_h c, wd, r0, r1, r2, r3 + umin \r0\wd, \r0\wd, \c\wd +.ifnb \r1 + umin \r1\wd, \r1\wd, \c\wd +.endif +.ifnb \r2 + umin \r2\wd, \r2\wd, \c\wd + umin \r3\wd, \r3\wd, \c\wd +.endif +.endm +.macro sub_h c, wd, r0, r1, r2, r3 + sub \r0\wd, \r0\wd, \c\wd +.ifnb \r1 + sub \r1\wd, \r1\wd, \c\wd +.endif +.ifnb \r2 + sub \r2\wd, \r2\wd, \c\wd + sub \r3\wd, \r3\wd, \c\wd +.endif +.endm +.macro smull_smlal_4 d, s0, s1, s2, s3 + smull \d\().4s, \s0\().4h, v0.h[0] + smlal \d\().4s, \s1\().4h, v0.h[1] + smlal \d\().4s, \s2\().4h, v0.h[2] + smlal \d\().4s, \s3\().4h, v0.h[3] +.endm +.macro smull2_smlal2_4 d, s0, s1, s2, s3 + smull2 \d\().4s, \s0\().8h, v0.h[0] + smlal2 \d\().4s, \s1\().8h, v0.h[1] + smlal2 \d\().4s, \s2\().8h, v0.h[2] + smlal2 \d\().4s, \s3\().8h, v0.h[3] +.endm +.macro smull_smlal_8 d, s0, s1, s2, s3, s4, s5, s6, s7 + smull \d\().4s, \s0\().4h, v0.h[0] + smlal \d\().4s, \s1\().4h, v0.h[1] + smlal \d\().4s, \s2\().4h, v0.h[2] + smlal \d\().4s, \s3\().4h, v0.h[3] + smlal \d\().4s, \s4\().4h, v0.h[4] + smlal \d\().4s, \s5\().4h, v0.h[5] + smlal \d\().4s, \s6\().4h, v0.h[6] + smlal \d\().4s, \s7\().4h, v0.h[7] +.endm +.macro smull2_smlal2_8 d, s0, s1, s2, s3, s4, s5, s6, s7 + smull2 \d\().4s, \s0\().8h, v0.h[0] + smlal2 \d\().4s, \s1\().8h, v0.h[1] + smlal2 \d\().4s, \s2\().8h, v0.h[2] + smlal2 \d\().4s, \s3\().8h, v0.h[3] + smlal2 \d\().4s, \s4\().8h, v0.h[4] + smlal2 \d\().4s, \s5\().8h, v0.h[5] + smlal2 \d\().4s, \s6\().8h, v0.h[6] + smlal2 \d\().4s, \s7\().8h, v0.h[7] +.endm +.macro sqrshrun_h shift, r0, r1, r2, r3 + sqrshrun \r0\().4h, \r0\().4s, #\shift +.ifnb \r1 + sqrshrun2 \r0\().8h, \r1\().4s, #\shift +.endif +.ifnb \r2 + sqrshrun \r2\().4h, \r2\().4s, #\shift + sqrshrun2 \r2\().8h, \r3\().4s, #\shift +.endif +.endm +.macro xtn_h r0, r1, r2, r3 + xtn \r0\().4h, \r0\().4s + xtn2 \r0\().8h, \r1\().4s +.ifnb \r2 + xtn \r2\().4h, \r2\().4s + xtn2 \r2\().8h, \r3\().4s +.endif +.endm +.macro srshl_s shift, r0, r1, r2, r3 + srshl \r0\().4s, \r0\().4s, \shift\().4s + srshl \r1\().4s, \r1\().4s, \shift\().4s +.ifnb \r2 + srshl \r2\().4s, \r2\().4s, \shift\().4s + srshl \r3\().4s, \r3\().4s, \shift\().4s +.endif +.endm +.macro st_s strd, reg, lanes + st1 {\reg\().s}[0], [x0], \strd + st1 {\reg\().s}[1], [x9], \strd +.if \lanes > 2 + st1 {\reg\().s}[2], [x0], \strd + st1 {\reg\().s}[3], [x9], \strd +.endif +.endm +.macro st_d strd, r0, r1 + st1 {\r0\().d}[0], [x0], \strd + st1 {\r0\().d}[1], [x9], \strd +.ifnb \r1 + st1 {\r1\().d}[0], [x0], \strd + st1 {\r1\().d}[1], [x9], \strd +.endif +.endm +.macro shift_store_4 type, strd, r0, r1, r2, r3 +.ifc \type, put + sqrshrun_h 6, \r0, \r1, \r2, \r3 + umin_h v31, .8h, \r0, \r2 +.else + srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) + xtn_h \r0, \r1, \r2, \r3 + sub_h v29, .8h, \r0, \r2 // PREP_BIAS +.endif + st_d \strd, \r0, \r2 +.endm +.macro st_reg strd, wd, r0, r1, r2, r3, r4, r5, r6, r7 + st1 {\r0\wd}, [x0], \strd + st1 {\r1\wd}, [x9], \strd +.ifnb \r2 + st1 {\r2\wd}, [x0], \strd + st1 {\r3\wd}, [x9], \strd +.endif +.ifnb \r4 + st1 {\r4\wd}, [x0], \strd + st1 {\r5\wd}, [x9], \strd + st1 {\r6\wd}, [x0], \strd + st1 {\r7\wd}, [x9], \strd +.endif +.endm +.macro st_8h strd, r0, r1, r2, r3, r4, r5, r6, r7 + st_reg \strd, .8h, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endm +.macro shift_store_8 type, strd, r0, r1, r2, r3 +.ifc \type, put + sqrshrun_h 6, \r0, \r1, \r2, \r3 + umin_h v31, .8h, \r0, \r2 +.else + srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) + xtn_h \r0, \r1, \r2, \r3 + sub_h v29, .8h, \r0, \r2 // PREP_BIAS +.endif + st_8h \strd, \r0, \r2 +.endm +.macro shift_store_16 type, strd, dst, r0, r1, r2, r3 +.ifc \type, put + sqrshrun_h 6, \r0, \r1, \r2, \r3 + umin \r0\().8h, \r0\().8h, v31.8h + umin \r1\().8h, \r2\().8h, v31.8h +.else + srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) + xtn_h \r0, \r1, \r2, \r3 + sub \r0\().8h, \r0\().8h, v29.8h + sub \r1\().8h, \r2\().8h, v29.8h +.endif + st1 {\r0\().8h, \r1\().8h}, [\dst], \strd +.endm + +.macro make_8tap_fn op, type, type_h, type_v +function \op\()_8tap_\type\()_16bpc_neon, export=1 + mov w9, \type_h + mov w10, \type_v + b \op\()_8tap_neon +endfunc +.endm + +// No spaces in these expressions, due to gas-preprocessor. +#define REGULAR ((0*15<<7)|3*15) +#define SMOOTH ((1*15<<7)|4*15) +#define SHARP ((2*15<<7)|3*15) + +.macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, xmx, my, xmy, bdmax, ds2, sr2 +make_8tap_fn \type, regular, REGULAR, REGULAR +make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH +make_8tap_fn \type, regular_sharp, REGULAR, SHARP +make_8tap_fn \type, smooth, SMOOTH, SMOOTH +make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR +make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP +make_8tap_fn \type, sharp, SHARP, SHARP +make_8tap_fn \type, sharp_regular, SHARP, REGULAR +make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH + +function \type\()_8tap_neon +.ifc \bdmax, w8 + ldr w8, [sp] +.endif + mov w11, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) + mul \mx, \mx, w11 + mul \my, \my, w11 + add \mx, \mx, w9 // mx, 8tap_h, 4tap_h + add \my, \my, w10 // my, 8tap_v, 4tap_v +.ifc \type, prep + uxtw \d_strd, \w + lsl \d_strd, \d_strd, #1 +.endif + + dup v31.8h, \bdmax // bitdepth_max + clz \bdmax, \bdmax + clz w9, \w + sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 + mov w12, #6 + tst \mx, #(0x7f << 14) + sub w9, w9, #24 + add w13, w12, \bdmax // 6 + intermediate_bits + sub w12, w12, \bdmax // 6 - intermediate_bits + movrel x11, X(mc_subpel_filters), -8 + b.ne L(\type\()_8tap_h) + tst \my, #(0x7f << 14) + b.ne L(\type\()_8tap_v) + b \type\()_neon + +L(\type\()_8tap_h): + cmp \w, #4 + ubfx w10, \mx, #7, #7 + and \mx, \mx, #0x7f + b.le 4f + mov \mx, w10 +4: + tst \my, #(0x7f << 14) + add \xmx, x11, \mx, uxtw #3 + b.ne L(\type\()_8tap_hv) + + adr x10, L(\type\()_8tap_h_tbl) + dup v30.4s, w12 // 6 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v30.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + dup v29.8h, \bdmax // intermediate_bits +.else + movi v28.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v29.8h, v29.8h // -intermediate_bits +.endif + br x10 + +20: // 2xN h +.ifc \type, put + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + sub \src, \src, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b +2: + ld1 {v4.8h}, [\src], \s_strd + ld1 {v6.8h}, [\sr2], \s_strd + ext v5.16b, v4.16b, v4.16b, #2 + ext v7.16b, v6.16b, v6.16b, #2 + subs \h, \h, #2 + trn1 v3.2s, v4.2s, v6.2s + trn2 v6.2s, v4.2s, v6.2s + trn1 v4.2s, v5.2s, v7.2s + trn2 v7.2s, v5.2s, v7.2s + smull v3.4s, v3.4h, v0.h[0] + smlal v3.4s, v4.4h, v0.h[1] + smlal v3.4s, v6.4h, v0.h[2] + smlal v3.4s, v7.4h, v0.h[3] + srshl v3.4s, v3.4s, v30.4s // -(6-intermediate_bits) + sqxtun v3.4h, v3.4s + srshl v3.4h, v3.4h, v29.4h // -intermediate_bits + umin v3.4h, v3.4h, v31.4h + st1 {v3.s}[0], [\dst], \d_strd + st1 {v3.s}[1], [\ds2], \d_strd + b.gt 2b + ret +.endif + +40: // 4xN h + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + sub \src, \src, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b +4: + ld1 {v16.8h}, [\src], \s_strd + ld1 {v20.8h}, [\sr2], \s_strd + ext v17.16b, v16.16b, v16.16b, #2 + ext v18.16b, v16.16b, v16.16b, #4 + ext v19.16b, v16.16b, v16.16b, #6 + ext v21.16b, v20.16b, v20.16b, #2 + ext v22.16b, v20.16b, v20.16b, #4 + ext v23.16b, v20.16b, v20.16b, #6 + subs \h, \h, #2 + smull v16.4s, v16.4h, v0.h[0] + smlal v16.4s, v17.4h, v0.h[1] + smlal v16.4s, v18.4h, v0.h[2] + smlal v16.4s, v19.4h, v0.h[3] + smull v20.4s, v20.4h, v0.h[0] + smlal v20.4s, v21.4h, v0.h[1] + smlal v20.4s, v22.4h, v0.h[2] + smlal v20.4s, v23.4h, v0.h[3] + srshl v16.4s, v16.4s, v30.4s // -(6-intermediate_bits) + srshl v20.4s, v20.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + sqxtun v16.4h, v16.4s + sqxtun2 v16.8h, v20.4s + srshl v16.8h, v16.8h, v29.8h // -intermediate_bits + umin v16.8h, v16.8h, v31.8h +.else + xtn v16.4h, v16.4s + xtn2 v16.8h, v20.4s + sub v16.8h, v16.8h, v28.8h // PREP_BIAS +.endif + st1 {v16.d}[0], [\dst], \d_strd + st1 {v16.d}[1], [\ds2], \d_strd + b.gt 4b + ret + +80: +160: +320: +640: +1280: // 8xN, 16xN, 32xN, ... h + ld1 {v0.8b}, [\xmx] + sub \src, \src, #6 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b + + sub \s_strd, \s_strd, \w, uxtw #1 + sub \s_strd, \s_strd, #16 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, uxtw #1 +.endif +81: + ld1 {v16.8h, v17.8h}, [\src], #32 + ld1 {v20.8h, v21.8h}, [\sr2], #32 + mov \mx, \w + +8: + smull v18.4s, v16.4h, v0.h[0] + smull2 v19.4s, v16.8h, v0.h[0] + smull v22.4s, v20.4h, v0.h[0] + smull2 v23.4s, v20.8h, v0.h[0] +.irpc i, 1234567 + ext v24.16b, v16.16b, v17.16b, #(2*\i) + ext v25.16b, v20.16b, v21.16b, #(2*\i) + smlal v18.4s, v24.4h, v0.h[\i] + smlal2 v19.4s, v24.8h, v0.h[\i] + smlal v22.4s, v25.4h, v0.h[\i] + smlal2 v23.4s, v25.8h, v0.h[\i] +.endr + subs \mx, \mx, #8 + srshl v18.4s, v18.4s, v30.4s // -(6-intermediate_bits) + srshl v19.4s, v19.4s, v30.4s // -(6-intermediate_bits) + srshl v22.4s, v22.4s, v30.4s // -(6-intermediate_bits) + srshl v23.4s, v23.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + sqxtun v18.4h, v18.4s + sqxtun2 v18.8h, v19.4s + sqxtun v22.4h, v22.4s + sqxtun2 v22.8h, v23.4s + srshl v18.8h, v18.8h, v29.8h // -intermediate_bits + srshl v22.8h, v22.8h, v29.8h // -intermediate_bits + umin v18.8h, v18.8h, v31.8h + umin v22.8h, v22.8h, v31.8h +.else + xtn v18.4h, v18.4s + xtn2 v18.8h, v19.4s + xtn v22.4h, v22.4s + xtn2 v22.8h, v23.4s + sub v18.8h, v18.8h, v28.8h // PREP_BIAS + sub v22.8h, v22.8h, v28.8h // PREP_BIAS +.endif + st1 {v18.8h}, [\dst], #16 + st1 {v22.8h}, [\ds2], #16 + b.le 9f + + mov v16.16b, v17.16b + mov v20.16b, v21.16b + ld1 {v17.8h}, [\src], #16 + ld1 {v21.8h}, [\sr2], #16 + b 8b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + b.gt 81b + ret + +L(\type\()_8tap_h_tbl): + .hword L(\type\()_8tap_h_tbl) - 1280b + .hword L(\type\()_8tap_h_tbl) - 640b + .hword L(\type\()_8tap_h_tbl) - 320b + .hword L(\type\()_8tap_h_tbl) - 160b + .hword L(\type\()_8tap_h_tbl) - 80b + .hword L(\type\()_8tap_h_tbl) - 40b + .hword L(\type\()_8tap_h_tbl) - 20b + .hword 0 + + +L(\type\()_8tap_v): + cmp \h, #4 + ubfx w10, \my, #7, #7 + and \my, \my, #0x7f + b.le 4f + mov \my, w10 +4: + add \xmy, x11, \my, uxtw #3 + +.ifc \type, prep + dup v30.4s, w12 // 6 - intermediate_bits + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + adr x10, L(\type\()_8tap_v_tbl) + ldrh w9, [x10, x9, lsl #1] +.ifc \type, prep + neg v30.4s, v30.4s // -(6-intermediate_bits) +.endif + sub x10, x10, w9, uxtw + br x10 + +20: // 2xN v +.ifc \type, put + b.gt 28f + + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + // 2x2 v + load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + interleave_1_s v1, v2, v3, v4, v5 + b.gt 24f + smull_smlal_4 v6, v1, v2, v3, v4 + sqrshrun_h 6, v6 + umin_h v31, .8h, v6 + st_s \d_strd, v6, 2 + ret + +24: // 2x4 v + load_s \sr2, \src, \s_strd, v6, v7 + interleave_1_s v5, v6, v7 + smull_smlal_4 v16, v1, v2, v3, v4 + smull_smlal_4 v17, v3, v4, v5, v6 + sqrshrun_h 6, v16, v17 + umin_h v31, .8h, v16 + st_s \d_strd, v16, 4 + ret + +28: // 2x8, 2x16 v + ld1 {v0.8b}, [\xmy] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b + + load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5, v6, v7 + interleave_1_s v1, v2, v3, v4, v5 + interleave_1_s v5, v6, v7 +216: + subs \h, \h, #8 + load_s \sr2, \src, \s_strd, v16, v17, v18, v19 + load_s \sr2, \src, \s_strd, v20, v21, v22, v23 + interleave_1_s v7, v16, v17, v18, v19 + interleave_1_s v19, v20, v21, v22, v23 + smull_smlal_8 v24, v1, v2, v3, v4, v5, v6, v7, v16 + smull_smlal_8 v25, v3, v4, v5, v6, v7, v16, v17, v18 + smull_smlal_8 v26, v5, v6, v7, v16, v17, v18, v19, v20 + smull_smlal_8 v27, v7, v16, v17, v18, v19, v20, v21, v22 + sqrshrun_h 6, v24, v25, v26, v27 + umin_h v31, .8h, v24, v26 + st_s \d_strd, v24, 4 + st_s \d_strd, v26, 4 + b.le 0f + mov v1.16b, v17.16b + mov v2.16b, v18.16b + mov v3.16b, v19.16b + mov v4.16b, v20.16b + mov v5.16b, v21.16b + mov v6.16b, v22.16b + mov v7.16b, v23.16b + b 216b +0: + ret +.endif + +40: + b.gt 480f + + // 4x2, 4x4 v + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_4h \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + smull_smlal_4 v6, v1, v2, v3, v4 + smull_smlal_4 v7, v2, v3, v4, v5 + shift_store_4 \type, \d_strd, v6, v7 + b.le 0f + load_4h \sr2, \src, \s_strd, v6, v7 + smull_smlal_4 v1, v3, v4, v5, v6 + smull_smlal_4 v2, v4, v5, v6, v7 + shift_store_4 \type, \d_strd, v1, v2 +0: + ret + +480: // 4x8, 4x16 v + ld1 {v0.8b}, [\xmy] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_4h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 + +48: + subs \h, \h, #4 + load_4h \sr2, \src, \s_strd, v23, v24, v25, v26 + smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 + smull_smlal_8 v2, v17, v18, v19, v20, v21, v22, v23, v24 + smull_smlal_8 v3, v18, v19, v20, v21, v22, v23, v24, v25 + smull_smlal_8 v4, v19, v20, v21, v22, v23, v24, v25, v26 + shift_store_4 \type, \d_strd, v1, v2, v3, v4 + b.le 0f + mov v16.8b, v20.8b + mov v17.8b, v21.8b + mov v18.8b, v22.8b + mov v19.8b, v23.8b + mov v20.8b, v24.8b + mov v21.8b, v25.8b + mov v22.8b, v26.8b + b 48b +0: + ret + +80: + b.gt 880f + + // 8x2, 8x4 v + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_8h \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + smull_smlal_4 v16, v1, v2, v3, v4 + smull2_smlal2_4 v17, v1, v2, v3, v4 + smull_smlal_4 v18, v2, v3, v4, v5 + smull2_smlal2_4 v19, v2, v3, v4, v5 + shift_store_8 \type, \d_strd, v16, v17, v18, v19 + b.le 0f + load_8h \sr2, \src, \s_strd, v6, v7 + smull_smlal_4 v16, v3, v4, v5, v6 + smull2_smlal2_4 v17, v3, v4, v5, v6 + smull_smlal_4 v18, v4, v5, v6, v7 + smull2_smlal2_4 v19, v4, v5, v6, v7 + shift_store_8 \type, \d_strd, v16, v17, v18, v19 +0: + ret + +880: // 8x6, 8x8, 8x16, 8x32 v +1680: // 16x8, 16x16, ... +320: // 32x8, 32x16, ... +640: +1280: + ld1 {v0.8b}, [\xmy] + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + sxtl v0.8h, v0.8b + mov \my, \h +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + load_8h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 + +88: + subs \h, \h, #2 + load_8h \sr2, \src, \s_strd, v23, v24 + smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 + smull2_smlal2_8 v2, v16, v17, v18, v19, v20, v21, v22, v23 + smull_smlal_8 v3, v17, v18, v19, v20, v21, v22, v23, v24 + smull2_smlal2_8 v4, v17, v18, v19, v20, v21, v22, v23, v24 + shift_store_8 \type, \d_strd, v1, v2, v3, v4 + b.le 9f + subs \h, \h, #2 + load_8h \sr2, \src, \s_strd, v25, v26 + smull_smlal_8 v1, v18, v19, v20, v21, v22, v23, v24, v25 + smull2_smlal2_8 v2, v18, v19, v20, v21, v22, v23, v24, v25 + smull_smlal_8 v3, v19, v20, v21, v22, v23, v24, v25, v26 + smull2_smlal2_8 v4, v19, v20, v21, v22, v23, v24, v25, v26 + shift_store_8 \type, \d_strd, v1, v2, v3, v4 + b.le 9f + mov v16.16b, v20.16b + mov v17.16b, v21.16b + mov v18.16b, v22.16b + mov v19.16b, v23.16b + mov v20.16b, v24.16b + mov v21.16b, v25.16b + mov v22.16b, v26.16b + b 88b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 168b +0: + ret + +160: + b.gt 1680b + + // 16x2, 16x4 v + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + sxtl v0.8h, v0.8b + + load_16h \src, \src, \s_strd, v16, v17, v18, v19, v20, v21 +16: + load_16h \src, \src, \s_strd, v22, v23 + subs \h, \h, #1 + smull_smlal_4 v1, v16, v18, v20, v22 + smull2_smlal2_4 v2, v16, v18, v20, v22 + smull_smlal_4 v3, v17, v19, v21, v23 + smull2_smlal2_4 v4, v17, v19, v21, v23 + shift_store_16 \type, \d_strd, x0, v1, v2, v3, v4 + b.le 0f + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v20.16b, v22.16b + mov v21.16b, v23.16b + b 16b +0: + ret + +L(\type\()_8tap_v_tbl): + .hword L(\type\()_8tap_v_tbl) - 1280b + .hword L(\type\()_8tap_v_tbl) - 640b + .hword L(\type\()_8tap_v_tbl) - 320b + .hword L(\type\()_8tap_v_tbl) - 160b + .hword L(\type\()_8tap_v_tbl) - 80b + .hword L(\type\()_8tap_v_tbl) - 40b + .hword L(\type\()_8tap_v_tbl) - 20b + .hword 0 + +L(\type\()_8tap_hv): + cmp \h, #4 + ubfx w10, \my, #7, #7 + and \my, \my, #0x7f + b.le 4f + mov \my, w10 +4: + add \xmy, x11, \my, uxtw #3 + + adr x10, L(\type\()_8tap_hv_tbl) + dup v30.4s, w12 // 6 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v30.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + dup v29.4s, w13 // 6 + intermediate_bits +.else + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v29.4s, v29.4s // -(6+intermediate_bits) +.endif + br x10 + +20: +.ifc \type, put + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + b.gt 280f + add \xmy, \xmy, #2 + ld1 {v1.s}[0], [\xmy] + + // 2x2, 2x4 hv + sub \sr2, \src, #2 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + sxtl v1.4s, v1.4h + + ld1 {v27.8h}, [\src], \s_strd + ext v28.16b, v27.16b, v27.16b, #2 + smull v27.4s, v27.4h, v0.4h + smull v28.4s, v28.4h, v0.4h + addp v27.4s, v27.4s, v28.4s + addp v16.4s, v27.4s, v27.4s + srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits) + bl L(\type\()_8tap_filter_2) + + trn1 v16.2d, v16.2d, v24.2d + mov v17.16b, v24.16b + +2: + bl L(\type\()_8tap_filter_2) + + ext v18.16b, v17.16b, v24.16b, #8 + mov v19.16b, v24.16b + mul v2.4s, v16.4s, v1.s[0] + mla v2.4s, v17.4s, v1.s[1] + mla v2.4s, v18.4s, v1.s[2] + mla v2.4s, v19.4s, v1.s[3] + + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + umin v2.4h, v2.4h, v31.4h + subs \h, \h, #2 + st1 {v2.s}[0], [\dst], \d_strd + st1 {v2.s}[1], [\ds2], \d_strd + b.le 0f + mov v16.16b, v18.16b + mov v17.16b, v19.16b + b 2b + +280: // 2x8, 2x16, 2x32 hv + ld1 {v1.8b}, [\xmy] + sub \src, \src, #2 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + sxtl2 v2.4s, v1.8h + sxtl v1.4s, v1.4h + + ld1 {v27.8h}, [\src], \s_strd + ext v28.16b, v27.16b, v27.16b, #2 + smull v27.4s, v27.4h, v0.4h + smull v28.4s, v28.4h, v0.4h + addp v27.4s, v27.4s, v28.4s + addp v16.4s, v27.4s, v27.4s + srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits) + + bl L(\type\()_8tap_filter_2) + trn1 v16.2d, v16.2d, v24.2d + mov v17.16b, v24.16b + bl L(\type\()_8tap_filter_2) + ext v18.16b, v17.16b, v24.16b, #8 + mov v19.16b, v24.16b + bl L(\type\()_8tap_filter_2) + ext v20.16b, v19.16b, v24.16b, #8 + mov v21.16b, v24.16b + +28: + bl L(\type\()_8tap_filter_2) + ext v22.16b, v21.16b, v24.16b, #8 + mov v23.16b, v24.16b + mul v3.4s, v16.4s, v1.s[0] + mla v3.4s, v17.4s, v1.s[1] + mla v3.4s, v18.4s, v1.s[2] + mla v3.4s, v19.4s, v1.s[3] + mla v3.4s, v20.4s, v2.s[0] + mla v3.4s, v21.4s, v2.s[1] + mla v3.4s, v22.4s, v2.s[2] + mla v3.4s, v23.4s, v2.s[3] + + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + sqxtun v3.4h, v3.4s + umin v3.4h, v3.4h, v31.4h + subs \h, \h, #2 + st1 {v3.s}[0], [\dst], \d_strd + st1 {v3.s}[1], [\ds2], \d_strd + b.le 0f + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v20.16b, v22.16b + mov v21.16b, v23.16b + b 28b + +0: + br x15 + +L(\type\()_8tap_filter_2): + ld1 {v25.8h}, [\sr2], \s_strd + ld1 {v27.8h}, [\src], \s_strd + ext v26.16b, v25.16b, v25.16b, #2 + ext v28.16b, v27.16b, v27.16b, #2 + trn1 v24.2s, v25.2s, v27.2s + trn2 v27.2s, v25.2s, v27.2s + trn1 v25.2s, v26.2s, v28.2s + trn2 v28.2s, v26.2s, v28.2s + smull v24.4s, v24.4h, v0.h[0] + smlal v24.4s, v25.4h, v0.h[1] + smlal v24.4s, v27.4h, v0.h[2] + smlal v24.4s, v28.4h, v0.h[3] + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + ret +.endif + +40: + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + b.gt 480f + add \xmy, \xmy, #2 + ld1 {v1.s}[0], [\xmy] + sub \sr2, \src, #2 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + // 4x2, 4x4 hv + ld1 {v25.8h}, [\src], \s_strd + ext v26.16b, v25.16b, v25.16b, #2 + ext v27.16b, v25.16b, v25.16b, #4 + ext v28.16b, v25.16b, v25.16b, #6 + smull v25.4s, v25.4h, v0.h[0] + smlal v25.4s, v26.4h, v0.h[1] + smlal v25.4s, v27.4h, v0.h[2] + smlal v25.4s, v28.4h, v0.h[3] + srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53). + xtn v16.4h, v16.4s + + bl L(\type\()_8tap_filter_4) + mov v17.8b, v24.8b + mov v18.8b, v25.8b + +4: + bl L(\type\()_8tap_filter_4) + smull v2.4s, v16.4h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal v2.4s, v24.4h, v1.h[3] + smull v3.4s, v17.4h, v1.h[0] + smlal v3.4s, v18.4h, v1.h[1] + smlal v3.4s, v24.4h, v1.h[2] + smlal v3.4s, v25.4h, v1.h[3] +.ifc \type, put + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + umin v2.8h, v2.8h, v31.8h +.else + rshrn v2.4h, v2.4s, #6 + rshrn2 v2.8h, v3.4s, #6 + sub v2.8h, v2.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + + st1 {v2.d}[0], [\dst], \d_strd + st1 {v2.d}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v24.8b + mov v18.8b, v25.8b + b 4b + +480: // 4x8, 4x16, 4x32 hv + ld1 {v1.8b}, [\xmy] + sub \src, \src, #2 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + ld1 {v25.8h}, [\src], \s_strd + ext v26.16b, v25.16b, v25.16b, #2 + ext v27.16b, v25.16b, v25.16b, #4 + ext v28.16b, v25.16b, v25.16b, #6 + smull v25.4s, v25.4h, v0.h[0] + smlal v25.4s, v26.4h, v0.h[1] + smlal v25.4s, v27.4h, v0.h[2] + smlal v25.4s, v28.4h, v0.h[3] + srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53). + xtn v16.4h, v16.4s + + bl L(\type\()_8tap_filter_4) + mov v17.8b, v24.8b + mov v18.8b, v25.8b + bl L(\type\()_8tap_filter_4) + mov v19.8b, v24.8b + mov v20.8b, v25.8b + bl L(\type\()_8tap_filter_4) + mov v21.8b, v24.8b + mov v22.8b, v25.8b + +48: + bl L(\type\()_8tap_filter_4) + smull v3.4s, v16.4h, v1.h[0] + smlal v3.4s, v17.4h, v1.h[1] + smlal v3.4s, v18.4h, v1.h[2] + smlal v3.4s, v19.4h, v1.h[3] + smlal v3.4s, v20.4h, v1.h[4] + smlal v3.4s, v21.4h, v1.h[5] + smlal v3.4s, v22.4h, v1.h[6] + smlal v3.4s, v24.4h, v1.h[7] + smull v4.4s, v17.4h, v1.h[0] + smlal v4.4s, v18.4h, v1.h[1] + smlal v4.4s, v19.4h, v1.h[2] + smlal v4.4s, v20.4h, v1.h[3] + smlal v4.4s, v21.4h, v1.h[4] + smlal v4.4s, v22.4h, v1.h[5] + smlal v4.4s, v24.4h, v1.h[6] + smlal v4.4s, v25.4h, v1.h[7] +.ifc \type, put + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) + sqxtun v3.4h, v3.4s + sqxtun2 v3.8h, v4.4s + umin v3.8h, v3.8h, v31.8h +.else + rshrn v3.4h, v3.4s, #6 + rshrn2 v3.8h, v4.4s, #6 + sub v3.8h, v3.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + st1 {v3.d}[0], [\dst], \d_strd + st1 {v3.d}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v19.8b + mov v18.8b, v20.8b + mov v19.8b, v21.8b + mov v20.8b, v22.8b + mov v21.8b, v24.8b + mov v22.8b, v25.8b + b 48b +0: + br x15 + +L(\type\()_8tap_filter_4): + ld1 {v24.8h}, [\sr2], \s_strd + ld1 {v25.8h}, [\src], \s_strd + ext v26.16b, v24.16b, v24.16b, #2 + ext v27.16b, v24.16b, v24.16b, #4 + ext v28.16b, v24.16b, v24.16b, #6 + smull v24.4s, v24.4h, v0.h[0] + smlal v24.4s, v26.4h, v0.h[1] + smlal v24.4s, v27.4h, v0.h[2] + smlal v24.4s, v28.4h, v0.h[3] + ext v26.16b, v25.16b, v25.16b, #2 + ext v27.16b, v25.16b, v25.16b, #4 + ext v28.16b, v25.16b, v25.16b, #6 + smull v25.4s, v25.4h, v0.h[0] + smlal v25.4s, v26.4h, v0.h[1] + smlal v25.4s, v27.4h, v0.h[2] + smlal v25.4s, v28.4h, v0.h[3] + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + xtn v24.4h, v24.4s + xtn v25.4h, v25.4s + ret + +80: +160: +320: + b.gt 880f + add \xmy, \xmy, #2 + ld1 {v0.8b}, [\xmx] + ld1 {v1.s}[0], [\xmy] + sub \src, \src, #6 + sub \src, \src, \s_strd + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + mov \my, \h + +164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + ld1 {v27.8h, v28.8h}, [\src], \s_strd + smull v24.4s, v27.4h, v0.h[0] + smull2 v25.4s, v27.8h, v0.h[0] +.irpc i, 1234567 + ext v26.16b, v27.16b, v28.16b, #(2*\i) + smlal v24.4s, v26.4h, v0.h[\i] + smlal2 v25.4s, v26.8h, v0.h[\i] +.endr + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53), + // and conserves register space (no need to clobber v8-v15). + xtn v16.4h, v24.4s + xtn2 v16.8h, v25.4s + + bl L(\type\()_8tap_filter_8) + mov v17.16b, v23.16b + mov v18.16b, v24.16b + +8: + smull v2.4s, v16.4h, v1.h[0] + smull2 v3.4s, v16.8h, v1.h[0] + bl L(\type\()_8tap_filter_8) + smull v4.4s, v17.4h, v1.h[0] + smull2 v5.4s, v17.8h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal2 v3.4s, v17.8h, v1.h[1] + smlal v4.4s, v18.4h, v1.h[1] + smlal2 v5.4s, v18.8h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal2 v3.4s, v18.8h, v1.h[2] + smlal v4.4s, v23.4h, v1.h[2] + smlal2 v5.4s, v23.8h, v1.h[2] + smlal v2.4s, v23.4h, v1.h[3] + smlal2 v3.4s, v23.8h, v1.h[3] + smlal v4.4s, v24.4h, v1.h[3] + smlal2 v5.4s, v24.8h, v1.h[3] +.ifc \type, put + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) + srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + sqxtun v3.4h, v4.4s + sqxtun2 v3.8h, v5.4s + umin v2.8h, v2.8h, v31.8h + umin v3.8h, v3.8h, v31.8h +.else + rshrn v2.4h, v2.4s, #6 + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + sub v2.8h, v2.8h, v29.8h // PREP_BIAS + sub v3.8h, v3.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + st1 {v2.8h}, [\dst], \d_strd + st1 {v3.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v18.16b + mov v17.16b, v23.16b + mov v18.16b, v24.16b + b 8b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #2 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 164b + +880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv +640: +1280: + ld1 {v0.8b}, [\xmx] + ld1 {v1.8b}, [\xmy] + sub \src, \src, #6 + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + mov \my, \h + +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + ld1 {v27.8h, v28.8h}, [\src], \s_strd + smull v24.4s, v27.4h, v0.h[0] + smull2 v25.4s, v27.8h, v0.h[0] +.irpc i, 1234567 + ext v26.16b, v27.16b, v28.16b, #(2*\i) + smlal v24.4s, v26.4h, v0.h[\i] + smlal2 v25.4s, v26.8h, v0.h[\i] +.endr + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53), + // and conserves register space (no need to clobber v8-v15). + xtn v16.4h, v24.4s + xtn2 v16.8h, v25.4s + + bl L(\type\()_8tap_filter_8) + mov v17.16b, v23.16b + mov v18.16b, v24.16b + bl L(\type\()_8tap_filter_8) + mov v19.16b, v23.16b + mov v20.16b, v24.16b + bl L(\type\()_8tap_filter_8) + mov v21.16b, v23.16b + mov v22.16b, v24.16b + +88: + smull v2.4s, v16.4h, v1.h[0] + smull2 v3.4s, v16.8h, v1.h[0] + bl L(\type\()_8tap_filter_8) + smull v4.4s, v17.4h, v1.h[0] + smull2 v5.4s, v17.8h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal2 v3.4s, v17.8h, v1.h[1] + smlal v4.4s, v18.4h, v1.h[1] + smlal2 v5.4s, v18.8h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal2 v3.4s, v18.8h, v1.h[2] + smlal v4.4s, v19.4h, v1.h[2] + smlal2 v5.4s, v19.8h, v1.h[2] + smlal v2.4s, v19.4h, v1.h[3] + smlal2 v3.4s, v19.8h, v1.h[3] + smlal v4.4s, v20.4h, v1.h[3] + smlal2 v5.4s, v20.8h, v1.h[3] + smlal v2.4s, v20.4h, v1.h[4] + smlal2 v3.4s, v20.8h, v1.h[4] + smlal v4.4s, v21.4h, v1.h[4] + smlal2 v5.4s, v21.8h, v1.h[4] + smlal v2.4s, v21.4h, v1.h[5] + smlal2 v3.4s, v21.8h, v1.h[5] + smlal v4.4s, v22.4h, v1.h[5] + smlal2 v5.4s, v22.8h, v1.h[5] + smlal v2.4s, v22.4h, v1.h[6] + smlal2 v3.4s, v22.8h, v1.h[6] + smlal v4.4s, v23.4h, v1.h[6] + smlal2 v5.4s, v23.8h, v1.h[6] + smlal v2.4s, v23.4h, v1.h[7] + smlal2 v3.4s, v23.8h, v1.h[7] + smlal v4.4s, v24.4h, v1.h[7] + smlal2 v5.4s, v24.8h, v1.h[7] +.ifc \type, put + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) + srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + sqxtun v3.4h, v4.4s + sqxtun2 v3.8h, v5.4s + umin v2.8h, v2.8h, v31.8h + umin v3.8h, v3.8h, v31.8h +.else + rshrn v2.4h, v2.4s, #6 + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + sub v2.8h, v2.8h, v29.8h // PREP_BIAS + sub v3.8h, v3.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + st1 {v2.8h}, [\dst], \d_strd + st1 {v3.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v20.16b, v22.16b + mov v21.16b, v23.16b + mov v22.16b, v24.16b + b 88b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 168b +0: + br x15 + +L(\type\()_8tap_filter_8): + ld1 {v4.8h, v5.8h}, [\sr2], \s_strd + ld1 {v6.8h, v7.8h}, [\src], \s_strd + smull v25.4s, v4.4h, v0.h[0] + smull2 v26.4s, v4.8h, v0.h[0] + smull v27.4s, v6.4h, v0.h[0] + smull2 v28.4s, v6.8h, v0.h[0] +.irpc i, 1234567 + ext v23.16b, v4.16b, v5.16b, #(2*\i) + ext v24.16b, v6.16b, v7.16b, #(2*\i) + smlal v25.4s, v23.4h, v0.h[\i] + smlal2 v26.4s, v23.8h, v0.h[\i] + smlal v27.4s, v24.4h, v0.h[\i] + smlal2 v28.4s, v24.8h, v0.h[\i] +.endr + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + srshl v26.4s, v26.4s, v30.4s // -(6-intermediate_bits) + srshl v27.4s, v27.4s, v30.4s // -(6-intermediate_bits) + srshl v28.4s, v28.4s, v30.4s // -(6-intermediate_bits) + xtn v23.4h, v25.4s + xtn2 v23.8h, v26.4s + xtn v24.4h, v27.4s + xtn2 v24.8h, v28.4s + ret + +L(\type\()_8tap_hv_tbl): + .hword L(\type\()_8tap_hv_tbl) - 1280b + .hword L(\type\()_8tap_hv_tbl) - 640b + .hword L(\type\()_8tap_hv_tbl) - 320b + .hword L(\type\()_8tap_hv_tbl) - 160b + .hword L(\type\()_8tap_hv_tbl) - 80b + .hword L(\type\()_8tap_hv_tbl) - 40b + .hword L(\type\()_8tap_hv_tbl) - 20b + .hword 0 +endfunc + + +function \type\()_bilin_16bpc_neon, export=1 +.ifc \bdmax, w8 + ldr w8, [sp] +.endif + dup v1.8h, \mx + dup v3.8h, \my + mov w10, #16 + sub w9, w10, \mx + sub w10, w10, \my + dup v0.8h, w9 + dup v2.8h, w10 +.ifc \type, prep + uxtw \d_strd, \w + lsl \d_strd, \d_strd, #1 +.endif + + clz \bdmax, \bdmax // bitdepth_max + clz w9, \w + sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 + mov w11, #4 + sub w9, w9, #24 + sub w11, w11, \bdmax // 4 - intermediate_bits + add w12, \bdmax, #4 // 4 + intermediate_bits + cbnz \mx, L(\type\()_bilin_h) + cbnz \my, L(\type\()_bilin_v) + b \type\()_neon + +L(\type\()_bilin_h): + cbnz \my, L(\type\()_bilin_hv) + + adr x10, L(\type\()_bilin_h_tbl) + dup v31.8h, w11 // 4 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v31.8h, v31.8h // -(4-intermediate_bits) +.ifc \type, put + dup v30.8h, \bdmax // intermediate_bits +.else + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v30.8h, v30.8h // -intermediate_bits +.endif + br x10 + +20: // 2xN h +.ifc \type, put + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +2: + ld1 {v4.4h}, [\src], \s_strd + ld1 {v6.4h}, [\sr2], \s_strd + ext v5.8b, v4.8b, v4.8b, #2 + ext v7.8b, v6.8b, v6.8b, #2 + trn1 v4.2s, v4.2s, v6.2s + trn1 v5.2s, v5.2s, v7.2s + subs \h, \h, #2 + mul v4.4h, v4.4h, v0.4h + mla v4.4h, v5.4h, v1.4h + urshl v4.4h, v4.4h, v31.4h + urshl v4.4h, v4.4h, v30.4h + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd + b.gt 2b + ret +.endif + +40: // 4xN h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +4: + ld1 {v4.8h}, [\src], \s_strd + ld1 {v6.8h}, [\sr2], \s_strd + ext v5.16b, v4.16b, v4.16b, #2 + ext v7.16b, v6.16b, v6.16b, #2 + trn1 v4.2d, v4.2d, v6.2d + trn1 v5.2d, v5.2d, v7.2d + subs \h, \h, #2 + mul v4.8h, v4.8h, v0.8h + mla v4.8h, v5.8h, v1.8h + urshl v4.8h, v4.8h, v31.8h +.ifc \type, put + urshl v4.8h, v4.8h, v30.8h +.else + sub v4.8h, v4.8h, v29.8h +.endif + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd + b.gt 4b + ret + +80: // 8xN h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +8: + ldr h5, [\src, #16] + ldr h7, [\sr2, #16] + ld1 {v4.8h}, [\src], \s_strd + ld1 {v6.8h}, [\sr2], \s_strd + ext v5.16b, v4.16b, v5.16b, #2 + ext v7.16b, v6.16b, v7.16b, #2 + subs \h, \h, #2 + mul v4.8h, v4.8h, v0.8h + mla v4.8h, v5.8h, v1.8h + mul v6.8h, v6.8h, v0.8h + mla v6.8h, v7.8h, v1.8h + urshl v4.8h, v4.8h, v31.8h + urshl v6.8h, v6.8h, v31.8h +.ifc \type, put + urshl v4.8h, v4.8h, v30.8h + urshl v6.8h, v6.8h, v30.8h +.else + sub v4.8h, v4.8h, v29.8h + sub v6.8h, v6.8h, v29.8h +.endif + st1 {v4.8h}, [\dst], \d_strd + st1 {v6.8h}, [\ds2], \d_strd + b.gt 8b + ret +160: +320: +640: +1280: // 16xN, 32xN, ... h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + + sub \s_strd, \s_strd, \w, uxtw #1 + sub \s_strd, \s_strd, #16 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, uxtw #1 +.endif +161: + ld1 {v16.8h}, [\src], #16 + ld1 {v21.8h}, [\sr2], #16 + mov \mx, \w + +16: + ld1 {v17.8h, v18.8h}, [\src], #32 + ld1 {v22.8h, v23.8h}, [\sr2], #32 + ext v19.16b, v16.16b, v17.16b, #2 + ext v20.16b, v17.16b, v18.16b, #2 + ext v24.16b, v21.16b, v22.16b, #2 + ext v25.16b, v22.16b, v23.16b, #2 + mul v16.8h, v16.8h, v0.8h + mla v16.8h, v19.8h, v1.8h + mul v17.8h, v17.8h, v0.8h + mla v17.8h, v20.8h, v1.8h + mul v21.8h, v21.8h, v0.8h + mla v21.8h, v24.8h, v1.8h + mul v22.8h, v22.8h, v0.8h + mla v22.8h, v25.8h, v1.8h + urshl v16.8h, v16.8h, v31.8h + urshl v17.8h, v17.8h, v31.8h + urshl v21.8h, v21.8h, v31.8h + urshl v22.8h, v22.8h, v31.8h + subs \mx, \mx, #16 +.ifc \type, put + urshl v16.8h, v16.8h, v30.8h + urshl v17.8h, v17.8h, v30.8h + urshl v21.8h, v21.8h, v30.8h + urshl v22.8h, v22.8h, v30.8h +.else + sub v16.8h, v16.8h, v29.8h + sub v17.8h, v17.8h, v29.8h + sub v21.8h, v21.8h, v29.8h + sub v22.8h, v22.8h, v29.8h +.endif + st1 {v16.8h, v17.8h}, [\dst], #32 + st1 {v21.8h, v22.8h}, [\ds2], #32 + b.le 9f + + mov v16.16b, v18.16b + mov v21.16b, v23.16b + b 16b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + b.gt 161b + ret + +L(\type\()_bilin_h_tbl): + .hword L(\type\()_bilin_h_tbl) - 1280b + .hword L(\type\()_bilin_h_tbl) - 640b + .hword L(\type\()_bilin_h_tbl) - 320b + .hword L(\type\()_bilin_h_tbl) - 160b + .hword L(\type\()_bilin_h_tbl) - 80b + .hword L(\type\()_bilin_h_tbl) - 40b + .hword L(\type\()_bilin_h_tbl) - 20b + .hword 0 + + +L(\type\()_bilin_v): + cmp \h, #4 + adr x10, L(\type\()_bilin_v_tbl) +.ifc \type, prep + dup v31.8h, w11 // 4 - intermediate_bits +.endif + ldrh w9, [x10, x9, lsl #1] +.ifc \type, prep + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 + neg v31.8h, v31.8h // -(4-intermediate_bits) +.endif + sub x10, x10, w9, uxtw + br x10 + +20: // 2xN v +.ifc \type, put + cmp \h, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + // 2x2 v + ld1 {v16.s}[0], [\src], \s_strd + b.gt 24f + ld1 {v17.s}[0], [\sr2], \s_strd + ld1 {v18.s}[0], [\src], \s_strd + trn1 v16.2s, v16.2s, v17.2s + trn1 v17.2s, v17.2s, v18.2s + mul v4.4h, v16.4h, v2.4h + mla v4.4h, v17.4h, v3.4h + urshr v4.8h, v4.8h, #4 + st1 {v4.s}[0], [\dst] + st1 {v4.s}[1], [\ds2] + ret +24: // 2x4, 2x8, ... v + ld1 {v17.s}[0], [\sr2], \s_strd + ld1 {v18.s}[0], [\src], \s_strd + ld1 {v19.s}[0], [\sr2], \s_strd + ld1 {v20.s}[0], [\src], \s_strd + trn1 v16.2s, v16.2s, v17.2s + trn1 v17.2s, v17.2s, v18.2s + trn1 v18.2s, v18.2s, v19.2s + trn1 v19.2s, v19.2s, v20.2s + trn1 v16.2d, v16.2d, v18.2d + trn1 v17.2d, v17.2d, v19.2d + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + subs \h, \h, #4 + urshr v4.8h, v4.8h, #4 + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd + st1 {v4.s}[2], [\dst], \d_strd + st1 {v4.s}[3], [\ds2], \d_strd + b.le 0f + mov v16.8b, v20.8b + b 24b +0: + ret +.endif + +40: // 4xN v + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + ld1 {v16.4h}, [\src], \s_strd +4: + ld1 {v17.4h}, [\sr2], \s_strd + ld1 {v18.4h}, [\src], \s_strd + trn1 v16.2d, v16.2d, v17.2d + trn1 v17.2d, v17.2d, v18.2d + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + urshr v4.8h, v4.8h, #4 +.else + urshl v4.8h, v4.8h, v31.8h + sub v4.8h, v4.8h, v29.8h +.endif + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + b 4b +0: + ret + +80: // 8xN v + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + ld1 {v16.8h}, [\src], \s_strd +8: + ld1 {v17.8h}, [\sr2], \s_strd + ld1 {v18.8h}, [\src], \s_strd + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + mul v5.8h, v17.8h, v2.8h + mla v5.8h, v18.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + urshr v4.8h, v4.8h, #4 + urshr v5.8h, v5.8h, #4 +.else + urshl v4.8h, v4.8h, v31.8h + urshl v5.8h, v5.8h, v31.8h + sub v4.8h, v4.8h, v29.8h + sub v5.8h, v5.8h, v29.8h +.endif + st1 {v4.8h}, [\dst], \d_strd + st1 {v5.8h}, [\ds2], \d_strd + b.le 0f + mov v16.16b, v18.16b + b 8b +0: + ret + +160: // 16xN, 32xN, ... +320: +640: +1280: + mov \my, \h +1: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v16.8h, v17.8h}, [\src], \s_strd +2: + ld1 {v18.8h, v19.8h}, [\sr2], \s_strd + ld1 {v20.8h, v21.8h}, [\src], \s_strd + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v18.8h, v3.8h + mul v5.8h, v17.8h, v2.8h + mla v5.8h, v19.8h, v3.8h + mul v6.8h, v18.8h, v2.8h + mla v6.8h, v20.8h, v3.8h + mul v7.8h, v19.8h, v2.8h + mla v7.8h, v21.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + urshr v4.8h, v4.8h, #4 + urshr v5.8h, v5.8h, #4 + urshr v6.8h, v6.8h, #4 + urshr v7.8h, v7.8h, #4 +.else + urshl v4.8h, v4.8h, v31.8h + urshl v5.8h, v5.8h, v31.8h + urshl v6.8h, v6.8h, v31.8h + urshl v7.8h, v7.8h, v31.8h + sub v4.8h, v4.8h, v29.8h + sub v5.8h, v5.8h, v29.8h + sub v6.8h, v6.8h, v29.8h + sub v7.8h, v7.8h, v29.8h +.endif + st1 {v4.8h, v5.8h}, [\dst], \d_strd + st1 {v6.8h, v7.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v20.16b + mov v17.16b, v21.16b + b 2b +9: + subs \w, \w, #16 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #32 + add \dst, \dst, #32 + b 1b +0: + ret + +L(\type\()_bilin_v_tbl): + .hword L(\type\()_bilin_v_tbl) - 1280b + .hword L(\type\()_bilin_v_tbl) - 640b + .hword L(\type\()_bilin_v_tbl) - 320b + .hword L(\type\()_bilin_v_tbl) - 160b + .hword L(\type\()_bilin_v_tbl) - 80b + .hword L(\type\()_bilin_v_tbl) - 40b + .hword L(\type\()_bilin_v_tbl) - 20b + .hword 0 + +L(\type\()_bilin_hv): + adr x10, L(\type\()_bilin_hv_tbl) + dup v31.8h, w11 // 4 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v31.8h, v31.8h // -(4-intermediate_bits) +.ifc \type, put + dup v30.4s, w12 // 4 + intermediate_bits +.else + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v30.4s, v30.4s // -(4+intermediate_bits) +.endif + br x10 + +20: // 2xN hv +.ifc \type, put + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v20.4h}, [\src], \s_strd + ext v21.8b, v20.8b, v20.8b, #2 + mul v16.4h, v20.4h, v0.4h + mla v16.4h, v21.4h, v1.4h + urshl v16.4h, v16.4h, v31.4h + +2: + ld1 {v22.4h}, [\sr2], \s_strd + ld1 {v24.4h}, [\src], \s_strd + ext v23.8b, v22.8b, v22.8b, #2 + ext v25.8b, v24.8b, v24.8b, #2 + trn1 v22.2s, v22.2s, v24.2s + trn1 v23.2s, v23.2s, v25.2s + mul v17.4h, v22.4h, v0.4h + mla v17.4h, v23.4h, v1.4h + urshl v17.4h, v17.4h, v31.4h + + trn1 v16.2s, v16.2s, v17.2s + + umull v4.4s, v16.4h, v2.4h + umlal v4.4s, v17.4h, v3.4h + urshl v4.4s, v4.4s, v30.4s + xtn v4.4h, v4.4s + subs \h, \h, #2 + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd + b.le 0f + trn2 v16.2s, v17.2s, v17.2s + b 2b +0: + ret +.endif + +40: // 4xN hv + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v20.8h}, [\src], \s_strd + ext v21.16b, v20.16b, v20.16b, #2 + mul v16.4h, v20.4h, v0.4h + mla v16.4h, v21.4h, v1.4h + urshl v16.4h, v16.4h, v31.4h + +4: + ld1 {v22.8h}, [\sr2], \s_strd + ld1 {v24.8h}, [\src], \s_strd + ext v23.16b, v22.16b, v22.16b, #2 + ext v25.16b, v24.16b, v24.16b, #2 + trn1 v22.2d, v22.2d, v24.2d + trn1 v23.2d, v23.2d, v25.2d + mul v17.8h, v22.8h, v0.8h + mla v17.8h, v23.8h, v1.8h + urshl v17.8h, v17.8h, v31.8h + + trn1 v16.2d, v16.2d, v17.2d + + umull v4.4s, v16.4h, v2.4h + umlal v4.4s, v17.4h, v3.4h + umull2 v5.4s, v16.8h, v2.8h + umlal2 v5.4s, v17.8h, v3.8h +.ifc \type, put + urshl v4.4s, v4.4s, v30.4s + urshl v5.4s, v5.4s, v30.4s + xtn v4.4h, v4.4s + xtn2 v4.8h, v5.4s +.else + rshrn v4.4h, v4.4s, #4 + rshrn2 v4.8h, v5.4s, #4 + sub v4.8h, v4.8h, v29.8h +.endif + subs \h, \h, #2 + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd + b.le 0f + trn2 v16.2d, v17.2d, v17.2d + b 4b +0: + ret + +80: // 8xN, 16xN, ... hv +160: +320: +640: +1280: + mov \my, \h + +1: + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ldr h21, [\src, #16] + ld1 {v20.8h}, [\src], \s_strd + ext v21.16b, v20.16b, v21.16b, #2 + mul v16.8h, v20.8h, v0.8h + mla v16.8h, v21.8h, v1.8h + urshl v16.8h, v16.8h, v31.8h + +2: + ldr h23, [\sr2, #16] + ld1 {v22.8h}, [\sr2], \s_strd + ldr h25, [\src, #16] + ld1 {v24.8h}, [\src], \s_strd + ext v23.16b, v22.16b, v23.16b, #2 + ext v25.16b, v24.16b, v25.16b, #2 + mul v17.8h, v22.8h, v0.8h + mla v17.8h, v23.8h, v1.8h + mul v18.8h, v24.8h, v0.8h + mla v18.8h, v25.8h, v1.8h + urshl v17.8h, v17.8h, v31.8h + urshl v18.8h, v18.8h, v31.8h + + umull v4.4s, v16.4h, v2.4h + umlal v4.4s, v17.4h, v3.4h + umull2 v5.4s, v16.8h, v2.8h + umlal2 v5.4s, v17.8h, v3.8h + umull v6.4s, v17.4h, v2.4h + umlal v6.4s, v18.4h, v3.4h + umull2 v7.4s, v17.8h, v2.8h + umlal2 v7.4s, v18.8h, v3.8h +.ifc \type, put + urshl v4.4s, v4.4s, v30.4s + urshl v5.4s, v5.4s, v30.4s + urshl v6.4s, v6.4s, v30.4s + urshl v7.4s, v7.4s, v30.4s + xtn v4.4h, v4.4s + xtn2 v4.8h, v5.4s + xtn v5.4h, v6.4s + xtn2 v5.8h, v7.4s +.else + rshrn v4.4h, v4.4s, #4 + rshrn2 v4.8h, v5.4s, #4 + rshrn v5.4h, v6.4s, #4 + rshrn2 v5.8h, v7.4s, #4 + sub v4.8h, v4.8h, v29.8h + sub v5.8h, v5.8h, v29.8h +.endif + subs \h, \h, #2 + st1 {v4.8h}, [\dst], \d_strd + st1 {v5.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v18.16b + b 2b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 1b +0: + ret + +L(\type\()_bilin_hv_tbl): + .hword L(\type\()_bilin_hv_tbl) - 1280b + .hword L(\type\()_bilin_hv_tbl) - 640b + .hword L(\type\()_bilin_hv_tbl) - 320b + .hword L(\type\()_bilin_hv_tbl) - 160b + .hword L(\type\()_bilin_hv_tbl) - 80b + .hword L(\type\()_bilin_hv_tbl) - 40b + .hword L(\type\()_bilin_hv_tbl) - 20b + .hword 0 +endfunc +.endm + +filter_fn put, x0, x1, x2, x3, w4, w5, w6, x6, w7, x7, w8, x9, x10 +filter_fn prep, x0, x8, x1, x2, w3, w4, w5, x5, w6, x6, w7, x9, x10 + +.macro load_filter_row dst, src, inc + asr w13, \src, #10 + ldr \dst, [x11, w13, sxtw #3] + add \src, \src, \inc +.endm + +function warp_filter_horz_neon + add w12, w5, #512 + + ld1 {v16.8h, v17.8h}, [x2], x3 + + load_filter_row d0, w12, w7 + load_filter_row d1, w12, w7 + load_filter_row d2, w12, w7 + sxtl v0.8h, v0.8b + load_filter_row d3, w12, w7 + sxtl v1.8h, v1.8b + load_filter_row d4, w12, w7 + sxtl v2.8h, v2.8b + load_filter_row d5, w12, w7 + sxtl v3.8h, v3.8b + load_filter_row d6, w12, w7 + sxtl v4.8h, v4.8b + load_filter_row d7, w12, w7 + sxtl v5.8h, v5.8b + ext v18.16b, v16.16b, v17.16b, #2*1 + smull v8.4s, v16.4h, v0.4h + smull2 v9.4s, v16.8h, v0.8h + sxtl v6.8h, v6.8b + ext v19.16b, v16.16b, v17.16b, #2*2 + smull v10.4s, v18.4h, v1.4h + smull2 v11.4s, v18.8h, v1.8h + sxtl v7.8h, v7.8b + ext v20.16b, v16.16b, v17.16b, #2*3 + smull v0.4s, v19.4h, v2.4h + smull2 v1.4s, v19.8h, v2.8h + ext v21.16b, v16.16b, v17.16b, #2*4 + addp v8.4s, v8.4s, v9.4s + smull v2.4s, v20.4h, v3.4h + smull2 v3.4s, v20.8h, v3.8h + ext v22.16b, v16.16b, v17.16b, #2*5 + addp v9.4s, v10.4s, v11.4s + smull v10.4s, v21.4h, v4.4h + smull2 v11.4s, v21.8h, v4.8h + ext v23.16b, v16.16b, v17.16b, #2*6 + addp v0.4s, v0.4s, v1.4s + smull v18.4s, v22.4h, v5.4h + smull2 v19.4s, v22.8h, v5.8h + ext v16.16b, v16.16b, v17.16b, #2*7 + addp v1.4s, v2.4s, v3.4s + addp v2.4s, v10.4s, v11.4s + smull v20.4s, v23.4h, v6.4h + smull2 v21.4s, v23.8h, v6.8h + addp v3.4s, v18.4s, v19.4s + smull v22.4s, v16.4h, v7.4h + smull2 v23.4s, v16.8h, v7.8h + addp v4.4s, v20.4s, v21.4s + addp v5.4s, v22.4s, v23.4s + + addp v8.4s, v8.4s, v9.4s + addp v0.4s, v0.4s, v1.4s + addp v2.4s, v2.4s, v3.4s + addp v4.4s, v4.4s, v5.4s + + addp v16.4s, v8.4s, v0.4s + addp v17.4s, v2.4s, v4.4s + + add w5, w5, w8 + + srshl v16.4s, v16.4s, v14.4s // -(7 - intermediate_bits) + srshl v17.4s, v17.4s, v14.4s // -(7 - intermediate_bits) + + ret +endfunc + +// void dav1d_warp_affine_8x8_16bpc_neon( +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *const abcd, int mx, int my, +// const int bitdepth_max) +.macro warp t +function warp_affine_8x8\t\()_16bpc_neon, export=1 + stp d8, d9, [sp, #-0x40]! + stp d10, d11, [sp, #0x10] + stp d12, d13, [sp, #0x20] + stp d14, d15, [sp, #0x30] + +.ifb \t + dup v15.8h, w7 // bitdepth_max +.else + movi v15.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + clz w7, w7 + // intermediate_bits = clz(bitdepth_max) - 18 +.ifb \t + sub w8, w7, #11 // 7 + intermediate_bits = clz(bitdepth_max) - 18 + 7 +.endif + sub w7, w7, #25 // -(7 - intermediate_bits) +.ifb \t + neg w8, w8 // -(7 + intermediate_bits) +.endif + dup v14.4s, w7 // -(7 - intermediate_bits) +.ifb \t + dup v13.4s, w8 // -(7 + intermediate_bits) +.endif + + ldr x4, [x4] + sbfx x7, x4, #0, #16 + sbfx x8, x4, #16, #16 + sbfx x9, x4, #32, #16 + sbfx x4, x4, #48, #16 + mov w10, #8 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + sub x2, x2, #6 + movrel x11, X(mc_warp_filter), 64*8 + mov x15, x30 +.ifnb \t + lsl x1, x1, #1 +.endif + + bl warp_filter_horz_neon + xtn v24.4h, v16.4s + xtn2 v24.8h, v17.4s + bl warp_filter_horz_neon + xtn v25.4h, v16.4s + xtn2 v25.8h, v17.4s + bl warp_filter_horz_neon + xtn v26.4h, v16.4s + xtn2 v26.8h, v17.4s + bl warp_filter_horz_neon + xtn v27.4h, v16.4s + xtn2 v27.8h, v17.4s + bl warp_filter_horz_neon + xtn v28.4h, v16.4s + xtn2 v28.8h, v17.4s + bl warp_filter_horz_neon + xtn v29.4h, v16.4s + xtn2 v29.8h, v17.4s + bl warp_filter_horz_neon + xtn v30.4h, v16.4s + xtn2 v30.8h, v17.4s + +1: + add w14, w6, #512 + bl warp_filter_horz_neon + xtn v31.4h, v16.4s + xtn2 v31.8h, v17.4s + + load_filter_row d0, w14, w9 + load_filter_row d1, w14, w9 + load_filter_row d2, w14, w9 + load_filter_row d3, w14, w9 + load_filter_row d4, w14, w9 + load_filter_row d5, w14, w9 + load_filter_row d6, w14, w9 + load_filter_row d7, w14, w9 + transpose_8x8b v0, v1, v2, v3, v4, v5, v6, v7, v16, v17 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + sxtl v2.8h, v2.8b + sxtl v3.8h, v3.8b + sxtl v4.8h, v4.8b + sxtl v5.8h, v5.8b + sxtl v6.8h, v6.8b + sxtl v7.8h, v7.8b + + // This ordering of smull/smlal/smull2/smlal2 is highly + // beneficial for Cortex A53 here. + smull v16.4s, v24.4h, v0.4h + smlal v16.4s, v25.4h, v1.4h + smlal v16.4s, v26.4h, v2.4h + smlal v16.4s, v27.4h, v3.4h + smlal v16.4s, v28.4h, v4.4h + smlal v16.4s, v29.4h, v5.4h + smlal v16.4s, v30.4h, v6.4h + smlal v16.4s, v31.4h, v7.4h + smull2 v17.4s, v24.8h, v0.8h + smlal2 v17.4s, v25.8h, v1.8h + smlal2 v17.4s, v26.8h, v2.8h + smlal2 v17.4s, v27.8h, v3.8h + smlal2 v17.4s, v28.8h, v4.8h + smlal2 v17.4s, v29.8h, v5.8h + smlal2 v17.4s, v30.8h, v6.8h + smlal2 v17.4s, v31.8h, v7.8h + + mov v24.16b, v25.16b + mov v25.16b, v26.16b +.ifb \t + srshl v16.4s, v16.4s, v13.4s // -(7 + intermediate_bits) + srshl v17.4s, v17.4s, v13.4s // -(7 + intermediate_bits) +.else + rshrn v16.4h, v16.4s, #7 + rshrn2 v16.8h, v17.4s, #7 +.endif + mov v26.16b, v27.16b +.ifb \t + sqxtun v16.4h, v16.4s + sqxtun2 v16.8h, v17.4s +.else + sub v16.8h, v16.8h, v15.8h // PREP_BIAS +.endif + mov v27.16b, v28.16b + mov v28.16b, v29.16b +.ifb \t + umin v16.8h, v16.8h, v15.8h // bitdepth_max +.endif + mov v29.16b, v30.16b + mov v30.16b, v31.16b + subs w10, w10, #1 + st1 {v16.8h}, [x0], x1 + + add w6, w6, w4 + b.gt 1b + + ldp d14, d15, [sp, #0x30] + ldp d12, d13, [sp, #0x20] + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x40 + + br x15 +endfunc +.endm + +warp +warp t + +// void dav1d_emu_edge_16bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_16bpc_neon, export=1 + ldp x8, x9, [sp] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub x12, x3, #1 // ih - 1 + cmp x5, x3 + sub x13, x2, #1 // iw - 1 + csel x12, x12, x5, ge // min(y, ih - 1) + cmp x4, x2 + bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0) + csel x13, x13, x4, ge // min(x, iw - 1) + bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0) + madd x8, x12, x9, x8 // ref += iclip() * stride + add x8, x8, x13, lsl #1 // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add x10, x5, x1 // y + bh + neg x5, x5 // -y + sub x10, x10, x3 // y + bh - ih + sub x12, x1, #1 // bh - 1 + cmp x10, x1 + bic x5, x5, x5, asr #63 // max(-y, 0) + csel x10, x10, x12, lt // min(y + bh - ih, bh-1) + cmp x5, x1 + bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0) + csel x5, x5, x12, lt // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add x11, x4, x0 // x + bw + neg x4, x4 // -x + sub x11, x11, x2 // x + bw - iw + sub x13, x0, #1 // bw - 1 + cmp x11, x0 + bic x4, x4, x4, asr #63 // max(-x, 0) + csel x11, x11, x13, lt // min(x + bw - iw, bw-1) + cmp x4, x0 + bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0) + csel x4, x4, x13, lt // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub x1, x1, x5 // bh - top_ext + madd x6, x5, x7, x6 + sub x2, x0, x4 // bw - left_ext + sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext + sub x2, x2, x11 // center_w = bw - left_ext - right_ext + + mov x14, x6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + ld1r {v0.8h}, [x8] + mov x12, x6 // out = dst + mov x3, x4 + mov v1.16b, v0.16b +1: + subs x3, x3, #16 + st1 {v0.8h, v1.8h}, [x12], #32 + b.gt 1b +.endif + mov x13, x8 + add x12, x6, x4, lsl #1 // out = dst + left_ext + mov x3, x2 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x13], #64 + subs x3, x3, #32 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x12], #64 + b.gt 1b +.if \need_right + add x3, x8, x2, lsl #1 // in + center_w + sub x3, x3, #2 // in + center_w - 1 + add x12, x6, x4, lsl #1 // dst + left_ext + ld1r {v0.8h}, [x3] + add x12, x12, x2, lsl #1 // out = dst + left_ext + center_w + mov x3, x11 + mov v1.16b, v0.16b +1: + subs x3, x3, #16 + st1 {v0.8h, v1.8h}, [x12], #32 + b.gt 1b +.endif + + subs x1, x1, #1 // center_h-- + add x6, x6, x7 + add x8, x8, x9 + b.gt 0b +.endm + + cbz x4, 2f + // need_left + cbz x11, 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cbz x11, 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + + cbz x10, 3f + // need_bottom + sub x8, x6, x7 // ref = dst - stride + mov x4, x0 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x8], #64 + mov x3, x10 +2: + subs x3, x3, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 + b.gt 2b + msub x6, x7, x10, x6 // dst -= bottom_ext * stride + subs x4, x4, #32 // bw -= 32 + add x6, x6, #64 // dst += 32 + b.gt 1b + +3: + cbz x5, 3f + // need_top + msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x14], #64 + mov x3, x5 +2: + subs x3, x3, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 + b.gt 2b + msub x6, x7, x5, x6 // dst -= top_ext * stride + subs x0, x0, #32 // bw -= 32 + add x6, x6, #64 // dst += 32 + b.gt 1b + +3: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/msac.S b/third_party/dav1d/src/arm/64/msac.S --- a/third_party/dav1d/src/arm/64/msac.S +++ b/third_party/dav1d/src/arm/64/msac.S @@ -108,29 +108,14 @@ .if \n == 16 sshl \d1\sz, \s1\sz, \s3\sz .endif .endm -.macro umull_n d0, d1, d2, d3, s0, s1, s2, s3, n - umull \d0\().4s, \s0\().4h, \s2\().4h -.if \n >= 8 - umull2 \d1\().4s, \s0\().8h, \s2\().8h -.endif +.macro sqdmulh_n d0, d1, s0, s1, s2, s3, sz, n + sqdmulh \d0\sz, \s0\sz, \s2\sz .if \n == 16 - umull \d2\().4s, \s1\().4h, \s3\().4h - umull2 \d3\().4s, \s1\().8h, \s3\().8h -.endif -.endm - -.macro shrn_n d0, d1, s0, s1, s2, s3, shift, n - shrn \d0\().4h, \s0\().4s, \shift -.if \n >= 8 - shrn2 \d0\().8h, \s1\().4s, \shift -.endif -.if \n == 16 - shrn \d1\().4h, \s2\().4s, \shift - shrn2 \d1\().8h, \s3\().4s, \shift + sqdmulh \d1\sz, \s1\sz, \s3\sz .endif .endm .macro str_n idx0, idx1, dstreg, dstoff, n str q\idx0, [\dstreg, \dstoff] @@ -147,21 +132,23 @@ sub sp, sp, #48 add x8, x0, #RNG ld1_n v0, v1, x1, \sz, \n // cdf ld1r {v4\sz}, [x8] // rng movrel x9, coeffs, 30 + movi v31\sz, #0x7f, lsl #8 // 0x7f00 sub x9, x9, x2, lsl #1 - ushr_n v2, v3, v0, v1, #6, \sz, \n // cdf >> EC_PROB_SHIFT + mvni v30\sz, #0x3f // 0xffc0 + and v7\szb, v4\szb, v31\szb // rng & 0x7f00 str h4, [sp, #14] // store original u = s->rng - ushr v4\sz, v4\sz, #8 // r = rng >> 8 + and_n v2, v3, v0, v1, v30, v30, \szb, \n // cdf & 0xffc0 - umull_n v16, v17, v18, v19, v4, v4, v2, v3, \n // r * (cdf >> EC_PROB_SHIFT) ld1_n v4, v5, x9, \sz, \n // EC_MIN_PROB * (n_symbols - ret) - shrn_n v2, v3, v16, v17, v18, v19, #1, \n // v >>= 7 - EC_PROB_SHIFT + sqdmulh_n v6, v7, v2, v3, v7, v7, \sz, \n // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 add x8, x0, #DIF + 6 - add_n v4, v5, v2, v3, v4, v5, \sz, \n // v += EC_MIN_PROB * (n_symbols - ret) + add_n v4, v5, v2, v3, v4, v5, \sz, \n // v = cdf + EC_MIN_PROB * (n_symbols - ret) + add_n v4, v5, v6, v7, v4, v5, \sz, \n // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) ld1r {v6.8h}, [x8] // dif >> (EC_WIN_SIZE - 16) movrel x8, bits str_n 4, 5, sp, #16, \n // store v values to allow indexed access diff --git a/third_party/dav1d/src/arm/cdef_init_tmpl.c b/third_party/dav1d/src/arm/cdef_init_tmpl.c --- a/third_party/dav1d/src/arm/cdef_init_tmpl.c +++ b/third_party/dav1d/src/arm/cdef_init_tmpl.c @@ -25,46 +25,50 @@ */ #include "src/cpu.h" #include "src/cdef.h" -#if BITDEPTH == 8 -decl_cdef_dir_fn(dav1d_cdef_find_dir_neon); +#if BITDEPTH == 8 || ARCH_AARCH64 +decl_cdef_dir_fn(BF(dav1d_cdef_find_dir, neon)); + +void BF(dav1d_cdef_padding4, neon)(uint16_t *tmp, const pixel *src, + ptrdiff_t src_stride, const pixel (*left)[2], + const pixel *const top, int h, + enum CdefEdgeFlags edges); +void BF(dav1d_cdef_padding8, neon)(uint16_t *tmp, const pixel *src, + ptrdiff_t src_stride, const pixel (*left)[2], + const pixel *const top, int h, + enum CdefEdgeFlags edges); -void dav1d_cdef_padding4_neon(uint16_t *tmp, const pixel *src, - ptrdiff_t src_stride, const pixel (*left)[2], - /*const*/ pixel *const top[2], int h, - enum CdefEdgeFlags edges); -void dav1d_cdef_padding8_neon(uint16_t *tmp, const pixel *src, - ptrdiff_t src_stride, const pixel (*left)[2], - /*const*/ pixel *const top[2], int h, - enum CdefEdgeFlags edges); - -void dav1d_cdef_filter4_neon(pixel *dst, ptrdiff_t dst_stride, - const uint16_t *tmp, int pri_strength, - int sec_strength, int dir, int damping, int h); -void dav1d_cdef_filter8_neon(pixel *dst, ptrdiff_t dst_stride, - const uint16_t *tmp, int pri_strength, - int sec_strength, int dir, int damping, int h); +// Passing edges to this function, to allow it to switch to a more +// optimized version for fully edged cases. Using size_t for edges, +// to avoid ABI differences for passing more than one argument on the stack. +void BF(dav1d_cdef_filter4, neon)(pixel *dst, ptrdiff_t dst_stride, + const uint16_t *tmp, int pri_strength, + int sec_strength, int dir, int damping, int h, + size_t edges HIGHBD_DECL_SUFFIX); +void BF(dav1d_cdef_filter8, neon)(pixel *dst, ptrdiff_t dst_stride, + const uint16_t *tmp, int pri_strength, + int sec_strength, int dir, int damping, int h, + size_t edges HIGHBD_DECL_SUFFIX); #define DEFINE_FILTER(w, h, tmp_stride) \ static void \ cdef_filter_##w##x##h##_neon(pixel *dst, \ const ptrdiff_t stride, \ - const pixel (*left)[2], \ - /*const*/ pixel *const top[2], \ - const int pri_strength, \ - const int sec_strength, \ - const int dir, \ - const int damping, \ - const enum CdefEdgeFlags edges) \ + const pixel (*left)[2], const pixel *const top, \ + const int pri_strength, const int sec_strength, \ + const int dir, const int damping, \ + const enum CdefEdgeFlags edges \ + HIGHBD_DECL_SUFFIX) \ { \ - ALIGN_STK_16(uint16_t, tmp_buf, 12*tmp_stride + 8,); \ + ALIGN_STK_16(uint16_t, tmp_buf, 12 * tmp_stride + 8,); \ uint16_t *tmp = tmp_buf + 2 * tmp_stride + 8; \ - dav1d_cdef_padding##w##_neon(tmp, dst, stride, left, top, h, edges); \ - dav1d_cdef_filter##w##_neon(dst, stride, tmp, pri_strength, \ - sec_strength, dir, damping, h); \ + BF(dav1d_cdef_padding##w, neon)(tmp, dst, stride, left, top, h, edges); \ + BF(dav1d_cdef_filter##w, neon)(dst, stride, tmp, pri_strength, \ + sec_strength, dir, damping, h, edges \ + HIGHBD_TAIL_SUFFIX); \ } DEFINE_FILTER(8, 8, 16) DEFINE_FILTER(4, 8, 8) DEFINE_FILTER(4, 4, 8) @@ -74,12 +78,12 @@ COLD void bitfn(dav1d_cdef_dsp_init_arm)(Dav1dCdefDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if BITDEPTH == 8 - c->dir = dav1d_cdef_find_dir_neon; +#if BITDEPTH == 8 || ARCH_AARCH64 + c->dir = BF(dav1d_cdef_find_dir, neon); c->fb[0] = cdef_filter_8x8_neon; c->fb[1] = cdef_filter_4x8_neon; c->fb[2] = cdef_filter_4x4_neon; #endif } diff --git a/third_party/dav1d/src/arm/cpu.c b/third_party/dav1d/src/arm/cpu.c --- a/third_party/dav1d/src/arm/cpu.c +++ b/third_party/dav1d/src/arm/cpu.c @@ -64,11 +64,14 @@ return 1; } // if line is incomplete seek back to avoid splitting the search // string into two buffers if (!strchr(line, '\n') && strlen(line) > strlen(flag)) { - if (fseeko(file, -strlen(flag), SEEK_CUR)) + // use fseek since the 64 bit fseeko is only available since + // Android API level 24 and meson defines _FILE_OFFSET_BITS + // by default 64 + if (fseek(file, -strlen(flag), SEEK_CUR)) break; } } fclose(file); diff --git a/third_party/dav1d/src/arm/ipred_init_tmpl.c b/third_party/dav1d/src/arm/ipred_init_tmpl.c --- a/third_party/dav1d/src/arm/ipred_init_tmpl.c +++ b/third_party/dav1d/src/arm/ipred_init_tmpl.c @@ -25,58 +25,58 @@ */ #include "src/cpu.h" #include "src/ipred.h" -decl_angular_ipred_fn(dav1d_ipred_dc_neon); -decl_angular_ipred_fn(dav1d_ipred_dc_128_neon); -decl_angular_ipred_fn(dav1d_ipred_dc_top_neon); -decl_angular_ipred_fn(dav1d_ipred_dc_left_neon); -decl_angular_ipred_fn(dav1d_ipred_h_neon); -decl_angular_ipred_fn(dav1d_ipred_v_neon); -decl_angular_ipred_fn(dav1d_ipred_paeth_neon); -decl_angular_ipred_fn(dav1d_ipred_smooth_neon); -decl_angular_ipred_fn(dav1d_ipred_smooth_v_neon); -decl_angular_ipred_fn(dav1d_ipred_smooth_h_neon); -decl_angular_ipred_fn(dav1d_ipred_filter_neon); +decl_angular_ipred_fn(BF(dav1d_ipred_dc, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_128, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_top, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_left, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_h, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_v, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_paeth, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth_v, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth_h, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_filter, neon)); -decl_cfl_pred_fn(dav1d_ipred_cfl_neon); -decl_cfl_pred_fn(dav1d_ipred_cfl_128_neon); -decl_cfl_pred_fn(dav1d_ipred_cfl_top_neon); -decl_cfl_pred_fn(dav1d_ipred_cfl_left_neon); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_128, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_top, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_left, neon)); -decl_cfl_ac_fn(dav1d_ipred_cfl_ac_420_neon); -decl_cfl_ac_fn(dav1d_ipred_cfl_ac_422_neon); +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_420, neon)); +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_422, neon)); -decl_pal_pred_fn(dav1d_pal_pred_neon); +decl_pal_pred_fn(BF(dav1d_pal_pred, neon)); COLD void bitfn(dav1d_intra_pred_dsp_init_arm)(Dav1dIntraPredDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if BITDEPTH == 8 - c->intra_pred[DC_PRED] = dav1d_ipred_dc_neon; - c->intra_pred[DC_128_PRED] = dav1d_ipred_dc_128_neon; - c->intra_pred[TOP_DC_PRED] = dav1d_ipred_dc_top_neon; - c->intra_pred[LEFT_DC_PRED] = dav1d_ipred_dc_left_neon; - c->intra_pred[HOR_PRED] = dav1d_ipred_h_neon; - c->intra_pred[VERT_PRED] = dav1d_ipred_v_neon; +#if BITDEPTH == 8 || ARCH_AARCH64 + c->intra_pred[DC_PRED] = BF(dav1d_ipred_dc, neon); + c->intra_pred[DC_128_PRED] = BF(dav1d_ipred_dc_128, neon); + c->intra_pred[TOP_DC_PRED] = BF(dav1d_ipred_dc_top, neon); + c->intra_pred[LEFT_DC_PRED] = BF(dav1d_ipred_dc_left, neon); + c->intra_pred[HOR_PRED] = BF(dav1d_ipred_h, neon); + c->intra_pred[VERT_PRED] = BF(dav1d_ipred_v, neon); #if ARCH_AARCH64 - c->intra_pred[PAETH_PRED] = dav1d_ipred_paeth_neon; - c->intra_pred[SMOOTH_PRED] = dav1d_ipred_smooth_neon; - c->intra_pred[SMOOTH_V_PRED] = dav1d_ipred_smooth_v_neon; - c->intra_pred[SMOOTH_H_PRED] = dav1d_ipred_smooth_h_neon; - c->intra_pred[FILTER_PRED] = dav1d_ipred_filter_neon; + c->intra_pred[PAETH_PRED] = BF(dav1d_ipred_paeth, neon); + c->intra_pred[SMOOTH_PRED] = BF(dav1d_ipred_smooth, neon); + c->intra_pred[SMOOTH_V_PRED] = BF(dav1d_ipred_smooth_v, neon); + c->intra_pred[SMOOTH_H_PRED] = BF(dav1d_ipred_smooth_h, neon); + c->intra_pred[FILTER_PRED] = BF(dav1d_ipred_filter, neon); - c->cfl_pred[DC_PRED] = dav1d_ipred_cfl_neon; - c->cfl_pred[DC_128_PRED] = dav1d_ipred_cfl_128_neon; - c->cfl_pred[TOP_DC_PRED] = dav1d_ipred_cfl_top_neon; - c->cfl_pred[LEFT_DC_PRED] = dav1d_ipred_cfl_left_neon; + c->cfl_pred[DC_PRED] = BF(dav1d_ipred_cfl, neon); + c->cfl_pred[DC_128_PRED] = BF(dav1d_ipred_cfl_128, neon); + c->cfl_pred[TOP_DC_PRED] = BF(dav1d_ipred_cfl_top, neon); + c->cfl_pred[LEFT_DC_PRED] = BF(dav1d_ipred_cfl_left, neon); - c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_ipred_cfl_ac_420_neon; - c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_ipred_cfl_ac_422_neon; + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_ipred_cfl_ac_420, neon); + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_ipred_cfl_ac_422, neon); - c->pal_pred = dav1d_pal_pred_neon; + c->pal_pred = BF(dav1d_pal_pred, neon); #endif #endif } diff --git a/third_party/dav1d/src/arm/loopfilter_init_tmpl.c b/third_party/dav1d/src/arm/loopfilter_init_tmpl.c --- a/third_party/dav1d/src/arm/loopfilter_init_tmpl.c +++ b/third_party/dav1d/src/arm/loopfilter_init_tmpl.c @@ -26,22 +26,22 @@ */ #include "src/cpu.h" #include "src/loopfilter.h" -decl_loopfilter_sb_fn(dav1d_lpf_h_sb_y_neon); -decl_loopfilter_sb_fn(dav1d_lpf_v_sb_y_neon); -decl_loopfilter_sb_fn(dav1d_lpf_h_sb_uv_neon); -decl_loopfilter_sb_fn(dav1d_lpf_v_sb_uv_neon); +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_y, neon)); +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_y, neon)); +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_uv, neon)); +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_uv, neon)); COLD void bitfn(dav1d_loop_filter_dsp_init_arm)(Dav1dLoopFilterDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if BITDEPTH == 8 - c->loop_filter_sb[0][0] = dav1d_lpf_h_sb_y_neon; - c->loop_filter_sb[0][1] = dav1d_lpf_v_sb_y_neon; - c->loop_filter_sb[1][0] = dav1d_lpf_h_sb_uv_neon; - c->loop_filter_sb[1][1] = dav1d_lpf_v_sb_uv_neon; +#if BITDEPTH == 8 || ARCH_AARCH64 + c->loop_filter_sb[0][0] = BF(dav1d_lpf_h_sb_y, neon); + c->loop_filter_sb[0][1] = BF(dav1d_lpf_v_sb_y, neon); + c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, neon); + c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, neon); #endif } diff --git a/third_party/dav1d/src/arm/looprestoration_init_tmpl.c b/third_party/dav1d/src/arm/looprestoration_init_tmpl.c --- a/third_party/dav1d/src/arm/looprestoration_init_tmpl.c +++ b/third_party/dav1d/src/arm/looprestoration_init_tmpl.c @@ -27,240 +27,272 @@ #include "src/cpu.h" #include "src/looprestoration.h" #include "src/tables.h" -#if BITDEPTH == 8 -// This calculates things slightly differently than the reference C version. -// This version calculates roughly this: +#if BITDEPTH == 8 || ARCH_AARCH64 +// The 8bpc version calculates things slightly differently than the reference +// C version. That version calculates roughly this: // int16_t sum = 0; // for (int i = 0; i < 7; i++) // sum += src[idx] * fh[i]; -// int16_t sum2 = (src[x] << 7) - (1 << (BITDEPTH + 6)) + rounding_off_h; +// int16_t sum2 = (src[x] << 7) - (1 << (bitdepth + 6)) + rounding_off_h; // sum = iclip(sum + sum2, INT16_MIN, INT16_MAX) >> round_bits_h; -// sum += 2048; -void dav1d_wiener_filter_h_neon(int16_t *dst, const pixel (*left)[4], - const pixel *src, ptrdiff_t stride, - const int16_t fh[7], const intptr_t w, - int h, enum LrEdgeFlags edges); +// sum += 1 << (bitdepth + 6 - round_bits_h); +// Compared to the reference C version, this is the output of the first pass +// _subtracted_ by 1 << (bitdepth + 6 - round_bits_h) = 2048, i.e. +// with round_offset precompensated. +// The 16bpc version calculates things pretty much the same way as the +// reference C version, but with the end result subtracted by +// 1 << (bitdepth + 6 - round_bits_h). +void BF(dav1d_wiener_filter_h, neon)(int16_t *dst, const pixel (*left)[4], + const pixel *src, ptrdiff_t stride, + const int16_t fh[7], const intptr_t w, + int h, enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX); // This calculates things slightly differently than the reference C version. // This version calculates roughly this: // fv[3] += 128; // int32_t sum = 0; // for (int i = 0; i < 7; i++) // sum += mid[idx] * fv[i]; // sum = (sum + rounding_off_v) >> round_bits_v; // This function assumes that the width is a multiple of 8. -void dav1d_wiener_filter_v_neon(pixel *dst, ptrdiff_t stride, - const int16_t *mid, int w, int h, - const int16_t fv[7], enum LrEdgeFlags edges, - ptrdiff_t mid_stride); -void dav1d_copy_narrow_neon(pixel *dst, ptrdiff_t stride, - const pixel *src, int w, int h); +void BF(dav1d_wiener_filter_v, neon)(pixel *dst, ptrdiff_t stride, + const int16_t *mid, int w, int h, + const int16_t fv[7], enum LrEdgeFlags edges, + ptrdiff_t mid_stride HIGHBD_DECL_SUFFIX); +void BF(dav1d_copy_narrow, neon)(pixel *dst, ptrdiff_t stride, + const pixel *src, int w, int h); static void wiener_filter_neon(pixel *const dst, const ptrdiff_t dst_stride, const pixel (*const left)[4], const pixel *lpf, const ptrdiff_t lpf_stride, const int w, const int h, const int16_t fh[7], - const int16_t fv[7], const enum LrEdgeFlags edges) + const int16_t fv[7], const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX) { ALIGN_STK_16(int16_t, mid, 68 * 384,); int mid_stride = (w + 7) & ~7; // Horizontal filter - dav1d_wiener_filter_h_neon(&mid[2 * mid_stride], left, dst, dst_stride, - fh, w, h, edges); + BF(dav1d_wiener_filter_h, neon)(&mid[2 * mid_stride], left, dst, dst_stride, + fh, w, h, edges HIGHBD_TAIL_SUFFIX); if (edges & LR_HAVE_TOP) - dav1d_wiener_filter_h_neon(mid, NULL, lpf, lpf_stride, - fh, w, 2, edges); + BF(dav1d_wiener_filter_h, neon)(mid, NULL, lpf, lpf_stride, + fh, w, 2, edges HIGHBD_TAIL_SUFFIX); if (edges & LR_HAVE_BOTTOM) - dav1d_wiener_filter_h_neon(&mid[(2 + h) * mid_stride], NULL, - lpf + 6 * PXSTRIDE(lpf_stride), lpf_stride, - fh, w, 2, edges); + BF(dav1d_wiener_filter_h, neon)(&mid[(2 + h) * mid_stride], NULL, + lpf + 6 * PXSTRIDE(lpf_stride), + lpf_stride, fh, w, 2, edges + HIGHBD_TAIL_SUFFIX); // Vertical filter if (w >= 8) - dav1d_wiener_filter_v_neon(dst, dst_stride, &mid[2*mid_stride], - w & ~7, h, fv, edges, mid_stride * sizeof(*mid)); + BF(dav1d_wiener_filter_v, neon)(dst, dst_stride, &mid[2*mid_stride], + w & ~7, h, fv, edges, + mid_stride * sizeof(*mid) + HIGHBD_TAIL_SUFFIX); if (w & 7) { // For uneven widths, do a full 8 pixel wide filtering into a temp // buffer and copy out the narrow slice of pixels separately into dest. ALIGN_STK_16(pixel, tmp, 64 * 8,); - dav1d_wiener_filter_v_neon(tmp, w & 7, &mid[2*mid_stride + (w & ~7)], - w & 7, h, fv, edges, mid_stride * sizeof(*mid)); - dav1d_copy_narrow_neon(dst + (w & ~7), dst_stride, tmp, w & 7, h); + BF(dav1d_wiener_filter_v, neon)(tmp, (w & 7) * sizeof(pixel), + &mid[2*mid_stride + (w & ~7)], + w & 7, h, fv, edges, + mid_stride * sizeof(*mid) + HIGHBD_TAIL_SUFFIX); + BF(dav1d_copy_narrow, neon)(dst + (w & ~7), dst_stride, tmp, w & 7, h); } } -void dav1d_sgr_box3_h_neon(int32_t *sumsq, int16_t *sum, - const pixel (*left)[4], - const pixel *src, const ptrdiff_t stride, - const int w, const int h, - const enum LrEdgeFlags edges); +void BF(dav1d_sgr_box3_h, neon)(int32_t *sumsq, int16_t *sum, + const pixel (*left)[4], + const pixel *src, const ptrdiff_t stride, + const int w, const int h, + const enum LrEdgeFlags edges); void dav1d_sgr_box3_v_neon(int32_t *sumsq, int16_t *sum, const int w, const int h, const enum LrEdgeFlags edges); void dav1d_sgr_calc_ab1_neon(int32_t *a, int16_t *b, - const int w, const int h, const int strength); -void dav1d_sgr_finish_filter1_neon(coef *tmp, - const pixel *src, const ptrdiff_t stride, - const int32_t *a, const int16_t *b, - const int w, const int h); + const int w, const int h, const int strength, + const int bitdepth_max); +void BF(dav1d_sgr_finish_filter1, neon)(int16_t *tmp, + const pixel *src, const ptrdiff_t stride, + const int32_t *a, const int16_t *b, + const int w, const int h); /* filter with a 3x3 box (radius=1) */ -static void dav1d_sgr_filter1_neon(coef *tmp, +static void dav1d_sgr_filter1_neon(int16_t *tmp, const pixel *src, const ptrdiff_t stride, const pixel (*left)[4], const pixel *lpf, const ptrdiff_t lpf_stride, const int w, const int h, const int strength, - const enum LrEdgeFlags edges) + const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX) { ALIGN_STK_16(int32_t, sumsq_mem, (384 + 16) * 68 + 8,); int32_t *const sumsq = &sumsq_mem[(384 + 16) * 2 + 8], *const a = sumsq; ALIGN_STK_16(int16_t, sum_mem, (384 + 16) * 68 + 16,); int16_t *const sum = &sum_mem[(384 + 16) * 2 + 16], *const b = sum; - dav1d_sgr_box3_h_neon(sumsq, sum, left, src, stride, w, h, edges); + BF(dav1d_sgr_box3_h, neon)(sumsq, sum, left, src, stride, w, h, edges); if (edges & LR_HAVE_TOP) - dav1d_sgr_box3_h_neon(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], - NULL, lpf, lpf_stride, w, 2, edges); + BF(dav1d_sgr_box3_h, neon)(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], + NULL, lpf, lpf_stride, w, 2, edges); if (edges & LR_HAVE_BOTTOM) - dav1d_sgr_box3_h_neon(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], - NULL, lpf + 6 * PXSTRIDE(lpf_stride), - lpf_stride, w, 2, edges); + BF(dav1d_sgr_box3_h, neon)(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], + NULL, lpf + 6 * PXSTRIDE(lpf_stride), + lpf_stride, w, 2, edges); dav1d_sgr_box3_v_neon(sumsq, sum, w, h, edges); - dav1d_sgr_calc_ab1_neon(a, b, w, h, strength); - dav1d_sgr_finish_filter1_neon(tmp, src, stride, a, b, w, h); + dav1d_sgr_calc_ab1_neon(a, b, w, h, strength, BITDEPTH_MAX); + BF(dav1d_sgr_finish_filter1, neon)(tmp, src, stride, a, b, w, h); } -void dav1d_sgr_box5_h_neon(int32_t *sumsq, int16_t *sum, - const pixel (*left)[4], - const pixel *src, const ptrdiff_t stride, - const int w, const int h, - const enum LrEdgeFlags edges); +void BF(dav1d_sgr_box5_h, neon)(int32_t *sumsq, int16_t *sum, + const pixel (*left)[4], + const pixel *src, const ptrdiff_t stride, + const int w, const int h, + const enum LrEdgeFlags edges); void dav1d_sgr_box5_v_neon(int32_t *sumsq, int16_t *sum, const int w, const int h, const enum LrEdgeFlags edges); void dav1d_sgr_calc_ab2_neon(int32_t *a, int16_t *b, - const int w, const int h, const int strength); -void dav1d_sgr_finish_filter2_neon(coef *tmp, - const pixel *src, const ptrdiff_t stride, - const int32_t *a, const int16_t *b, - const int w, const int h); + const int w, const int h, const int strength, + const int bitdepth_max); +void BF(dav1d_sgr_finish_filter2, neon)(int16_t *tmp, + const pixel *src, const ptrdiff_t stride, + const int32_t *a, const int16_t *b, + const int w, const int h); /* filter with a 5x5 box (radius=2) */ -static void dav1d_sgr_filter2_neon(coef *tmp, +static void dav1d_sgr_filter2_neon(int16_t *tmp, const pixel *src, const ptrdiff_t stride, const pixel (*left)[4], const pixel *lpf, const ptrdiff_t lpf_stride, const int w, const int h, const int strength, - const enum LrEdgeFlags edges) + const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX) { ALIGN_STK_16(int32_t, sumsq_mem, (384 + 16) * 68 + 8,); int32_t *const sumsq = &sumsq_mem[(384 + 16) * 2 + 8], *const a = sumsq; ALIGN_STK_16(int16_t, sum_mem, (384 + 16) * 68 + 16,); int16_t *const sum = &sum_mem[(384 + 16) * 2 + 16], *const b = sum; - dav1d_sgr_box5_h_neon(sumsq, sum, left, src, stride, w, h, edges); + BF(dav1d_sgr_box5_h, neon)(sumsq, sum, left, src, stride, w, h, edges); if (edges & LR_HAVE_TOP) - dav1d_sgr_box5_h_neon(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], - NULL, lpf, lpf_stride, w, 2, edges); + BF(dav1d_sgr_box5_h, neon)(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], + NULL, lpf, lpf_stride, w, 2, edges); if (edges & LR_HAVE_BOTTOM) - dav1d_sgr_box5_h_neon(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], - NULL, lpf + 6 * PXSTRIDE(lpf_stride), - lpf_stride, w, 2, edges); + BF(dav1d_sgr_box5_h, neon)(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], + NULL, lpf + 6 * PXSTRIDE(lpf_stride), + lpf_stride, w, 2, edges); dav1d_sgr_box5_v_neon(sumsq, sum, w, h, edges); - dav1d_sgr_calc_ab2_neon(a, b, w, h, strength); - dav1d_sgr_finish_filter2_neon(tmp, src, stride, a, b, w, h); + dav1d_sgr_calc_ab2_neon(a, b, w, h, strength, BITDEPTH_MAX); + BF(dav1d_sgr_finish_filter2, neon)(tmp, src, stride, a, b, w, h); } -void dav1d_sgr_weighted1_neon(pixel *dst, const ptrdiff_t dst_stride, - const pixel *src, const ptrdiff_t src_stride, - const coef *t1, const int w, const int h, - const int wt); -void dav1d_sgr_weighted2_neon(pixel *dst, const ptrdiff_t dst_stride, - const pixel *src, const ptrdiff_t src_stride, - const coef *t1, const coef *t2, - const int w, const int h, - const int16_t wt[2]); +void BF(dav1d_sgr_weighted1, neon)(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *t1, const int w, const int h, + const int wt HIGHBD_DECL_SUFFIX); +void BF(dav1d_sgr_weighted2, neon)(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *t1, const int16_t *t2, + const int w, const int h, + const int16_t wt[2] HIGHBD_DECL_SUFFIX); static void sgr_filter_neon(pixel *const dst, const ptrdiff_t dst_stride, const pixel (*const left)[4], const pixel *lpf, const ptrdiff_t lpf_stride, const int w, const int h, const int sgr_idx, - const int16_t sgr_wt[7], const enum LrEdgeFlags edges) + const int16_t sgr_wt[7], const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX) { if (!dav1d_sgr_params[sgr_idx][0]) { - ALIGN_STK_16(coef, tmp, 64 * 384,); + ALIGN_STK_16(int16_t, tmp, 64 * 384,); dav1d_sgr_filter1_neon(tmp, dst, dst_stride, left, lpf, lpf_stride, - w, h, dav1d_sgr_params[sgr_idx][3], edges); + w, h, dav1d_sgr_params[sgr_idx][3], edges + HIGHBD_TAIL_SUFFIX); if (w >= 8) - dav1d_sgr_weighted1_neon(dst, dst_stride, dst, dst_stride, - tmp, w & ~7, h, (1 << 7) - sgr_wt[1]); + BF(dav1d_sgr_weighted1, neon)(dst, dst_stride, dst, dst_stride, + tmp, w & ~7, h, (1 << 7) - sgr_wt[1] + HIGHBD_TAIL_SUFFIX); if (w & 7) { // For uneven widths, do a full 8 pixel wide filtering into a temp // buffer and copy out the narrow slice of pixels separately into // dest. ALIGN_STK_16(pixel, stripe, 64 * 8,); - dav1d_sgr_weighted1_neon(stripe, w & 7, dst + (w & ~7), dst_stride, - tmp + (w & ~7), w & 7, h, - (1 << 7) - sgr_wt[1]); - dav1d_copy_narrow_neon(dst + (w & ~7), dst_stride, stripe, - w & 7, h); + BF(dav1d_sgr_weighted1, neon)(stripe, (w & 7) * sizeof(pixel), + dst + (w & ~7), dst_stride, + tmp + (w & ~7), w & 7, h, + (1 << 7) - sgr_wt[1] + HIGHBD_TAIL_SUFFIX); + BF(dav1d_copy_narrow, neon)(dst + (w & ~7), dst_stride, stripe, + w & 7, h); } } else if (!dav1d_sgr_params[sgr_idx][1]) { - ALIGN_STK_16(coef, tmp, 64 * 384,); + ALIGN_STK_16(int16_t, tmp, 64 * 384,); dav1d_sgr_filter2_neon(tmp, dst, dst_stride, left, lpf, lpf_stride, - w, h, dav1d_sgr_params[sgr_idx][2], edges); + w, h, dav1d_sgr_params[sgr_idx][2], edges + HIGHBD_TAIL_SUFFIX); if (w >= 8) - dav1d_sgr_weighted1_neon(dst, dst_stride, dst, dst_stride, - tmp, w & ~7, h, sgr_wt[0]); + BF(dav1d_sgr_weighted1, neon)(dst, dst_stride, dst, dst_stride, + tmp, w & ~7, h, sgr_wt[0] + HIGHBD_TAIL_SUFFIX); if (w & 7) { // For uneven widths, do a full 8 pixel wide filtering into a temp // buffer and copy out the narrow slice of pixels separately into // dest. ALIGN_STK_16(pixel, stripe, 64 * 8,); - dav1d_sgr_weighted1_neon(stripe, w & 7, dst + (w & ~7), dst_stride, - tmp + (w & ~7), w & 7, h, sgr_wt[0]); - dav1d_copy_narrow_neon(dst + (w & ~7), dst_stride, stripe, - w & 7, h); + BF(dav1d_sgr_weighted1, neon)(stripe, (w & 7) * sizeof(pixel), + dst + (w & ~7), dst_stride, + tmp + (w & ~7), w & 7, h, sgr_wt[0] + HIGHBD_TAIL_SUFFIX); + BF(dav1d_copy_narrow, neon)(dst + (w & ~7), dst_stride, stripe, + w & 7, h); } } else { - ALIGN_STK_16(coef, tmp1, 64 * 384,); - ALIGN_STK_16(coef, tmp2, 64 * 384,); + ALIGN_STK_16(int16_t, tmp1, 64 * 384,); + ALIGN_STK_16(int16_t, tmp2, 64 * 384,); dav1d_sgr_filter2_neon(tmp1, dst, dst_stride, left, lpf, lpf_stride, - w, h, dav1d_sgr_params[sgr_idx][2], edges); + w, h, dav1d_sgr_params[sgr_idx][2], edges + HIGHBD_TAIL_SUFFIX); dav1d_sgr_filter1_neon(tmp2, dst, dst_stride, left, lpf, lpf_stride, - w, h, dav1d_sgr_params[sgr_idx][3], edges); + w, h, dav1d_sgr_params[sgr_idx][3], edges + HIGHBD_TAIL_SUFFIX); const int16_t wt[2] = { sgr_wt[0], 128 - sgr_wt[0] - sgr_wt[1] }; if (w >= 8) - dav1d_sgr_weighted2_neon(dst, dst_stride, dst, dst_stride, - tmp1, tmp2, w & ~7, h, wt); + BF(dav1d_sgr_weighted2, neon)(dst, dst_stride, dst, dst_stride, + tmp1, tmp2, w & ~7, h, wt + HIGHBD_TAIL_SUFFIX); if (w & 7) { // For uneven widths, do a full 8 pixel wide filtering into a temp // buffer and copy out the narrow slice of pixels separately into // dest. ALIGN_STK_16(pixel, stripe, 64 * 8,); - dav1d_sgr_weighted2_neon(stripe, w & 7, dst + (w & ~7), dst_stride, - tmp1 + (w & ~7), tmp2 + (w & ~7), - w & 7, h, wt); - dav1d_copy_narrow_neon(dst + (w & ~7), dst_stride, stripe, - w & 7, h); + BF(dav1d_sgr_weighted2, neon)(stripe, (w & 7) * sizeof(pixel), + dst + (w & ~7), dst_stride, + tmp1 + (w & ~7), tmp2 + (w & ~7), + w & 7, h, wt HIGHBD_TAIL_SUFFIX); + BF(dav1d_copy_narrow, neon)(dst + (w & ~7), dst_stride, stripe, + w & 7, h); } } } #endif // BITDEPTH == 8 -COLD void bitfn(dav1d_loop_restoration_dsp_init_arm)(Dav1dLoopRestorationDSPContext *const c) { +COLD void bitfn(dav1d_loop_restoration_dsp_init_arm)(Dav1dLoopRestorationDSPContext *const c, int bpc) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if BITDEPTH == 8 +#if BITDEPTH == 8 || ARCH_AARCH64 c->wiener = wiener_filter_neon; - c->selfguided = sgr_filter_neon; + if (bpc <= 10) + c->selfguided = sgr_filter_neon; #endif } diff --git a/third_party/dav1d/src/arm/mc_init_tmpl.c b/third_party/dav1d/src/arm/mc_init_tmpl.c --- a/third_party/dav1d/src/arm/mc_init_tmpl.c +++ b/third_party/dav1d/src/arm/mc_init_tmpl.c @@ -28,56 +28,58 @@ #include "config.h" #include "src/mc.h" #include "src/cpu.h" -decl_mc_fn(dav1d_put_8tap_regular_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_regular_smooth_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_regular_sharp_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_smooth_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_smooth_regular_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_smooth_sharp_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_sharp_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_sharp_regular_8bpc_neon); -decl_mc_fn(dav1d_put_8tap_sharp_smooth_8bpc_neon); -decl_mc_fn(dav1d_put_bilin_8bpc_neon); +decl_mc_fn(BF(dav1d_put_8tap_regular, neon)); +decl_mc_fn(BF(dav1d_put_8tap_regular_smooth, neon)); +decl_mc_fn(BF(dav1d_put_8tap_regular_sharp, neon)); +decl_mc_fn(BF(dav1d_put_8tap_smooth, neon)); +decl_mc_fn(BF(dav1d_put_8tap_smooth_regular, neon)); +decl_mc_fn(BF(dav1d_put_8tap_smooth_sharp, neon)); +decl_mc_fn(BF(dav1d_put_8tap_sharp, neon)); +decl_mc_fn(BF(dav1d_put_8tap_sharp_regular, neon)); +decl_mc_fn(BF(dav1d_put_8tap_sharp_smooth, neon)); +decl_mc_fn(BF(dav1d_put_bilin, neon)); -decl_mct_fn(dav1d_prep_8tap_regular_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_regular_smooth_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_regular_sharp_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_smooth_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_smooth_regular_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_smooth_sharp_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_sharp_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_sharp_regular_8bpc_neon); -decl_mct_fn(dav1d_prep_8tap_sharp_smooth_8bpc_neon); -decl_mct_fn(dav1d_prep_bilin_8bpc_neon); +decl_mct_fn(BF(dav1d_prep_8tap_regular, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_regular_smooth, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_regular_sharp, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_smooth, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_smooth_regular, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_smooth_sharp, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_sharp, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_sharp_regular, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_sharp_smooth, neon)); +decl_mct_fn(BF(dav1d_prep_bilin, neon)); -decl_avg_fn(dav1d_avg_8bpc_neon); -decl_w_avg_fn(dav1d_w_avg_8bpc_neon); -decl_mask_fn(dav1d_mask_8bpc_neon); -decl_blend_fn(dav1d_blend_8bpc_neon); -decl_blend_dir_fn(dav1d_blend_h_8bpc_neon); -decl_blend_dir_fn(dav1d_blend_v_8bpc_neon); +decl_avg_fn(BF(dav1d_avg, neon)); +decl_w_avg_fn(BF(dav1d_w_avg, neon)); +decl_mask_fn(BF(dav1d_mask, neon)); +decl_blend_fn(BF(dav1d_blend, neon)); +decl_blend_dir_fn(BF(dav1d_blend_h, neon)); +decl_blend_dir_fn(BF(dav1d_blend_v, neon)); -decl_w_mask_fn(dav1d_w_mask_444_8bpc_neon); -decl_w_mask_fn(dav1d_w_mask_422_8bpc_neon); -decl_w_mask_fn(dav1d_w_mask_420_8bpc_neon); +decl_w_mask_fn(BF(dav1d_w_mask_444, neon)); +decl_w_mask_fn(BF(dav1d_w_mask_422, neon)); +decl_w_mask_fn(BF(dav1d_w_mask_420, neon)); -decl_warp8x8_fn(dav1d_warp_affine_8x8_8bpc_neon); -decl_warp8x8t_fn(dav1d_warp_affine_8x8t_8bpc_neon); +decl_warp8x8_fn(BF(dav1d_warp_affine_8x8, neon)); +decl_warp8x8t_fn(BF(dav1d_warp_affine_8x8t, neon)); + +decl_emu_edge_fn(BF(dav1d_emu_edge, neon)); void bitfn(dav1d_mc_dsp_init_arm)(Dav1dMCDSPContext *const c) { #define init_mc_fn(type, name, suffix) \ - c->mc[type] = dav1d_put_##name##_8bpc_##suffix + c->mc[type] = BF(dav1d_put_##name, suffix) #define init_mct_fn(type, name, suffix) \ - c->mct[type] = dav1d_prep_##name##_8bpc_##suffix + c->mct[type] = BF(dav1d_prep_##name, suffix) const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if BITDEPTH == 8 +#if BITDEPTH == 8 || ARCH_AARCH64 init_mc_fn (FILTER_2D_8TAP_REGULAR, 8tap_regular, neon); init_mc_fn (FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, neon); init_mc_fn (FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, neon); init_mc_fn (FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, neon); init_mc_fn (FILTER_2D_8TAP_SMOOTH, 8tap_smooth, neon); @@ -96,18 +98,19 @@ init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, neon); init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, neon); init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, neon); init_mct_fn(FILTER_2D_BILINEAR, bilin, neon); - c->avg = dav1d_avg_8bpc_neon; - c->w_avg = dav1d_w_avg_8bpc_neon; - c->mask = dav1d_mask_8bpc_neon; - c->blend = dav1d_blend_8bpc_neon; - c->blend_h = dav1d_blend_h_8bpc_neon; - c->blend_v = dav1d_blend_v_8bpc_neon; - c->w_mask[0] = dav1d_w_mask_444_8bpc_neon; - c->w_mask[1] = dav1d_w_mask_422_8bpc_neon; - c->w_mask[2] = dav1d_w_mask_420_8bpc_neon; - c->warp8x8 = dav1d_warp_affine_8x8_8bpc_neon; - c->warp8x8t = dav1d_warp_affine_8x8t_8bpc_neon; + c->avg = BF(dav1d_avg, neon); + c->w_avg = BF(dav1d_w_avg, neon); + c->mask = BF(dav1d_mask, neon); + c->blend = BF(dav1d_blend, neon); + c->blend_h = BF(dav1d_blend_h, neon); + c->blend_v = BF(dav1d_blend_v, neon); + c->w_mask[0] = BF(dav1d_w_mask_444, neon); + c->w_mask[1] = BF(dav1d_w_mask_422, neon); + c->w_mask[2] = BF(dav1d_w_mask_420, neon); + c->warp8x8 = BF(dav1d_warp_affine_8x8, neon); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, neon); + c->emu_edge = BF(dav1d_emu_edge, neon); #endif } diff --git a/third_party/dav1d/src/cdef.h b/third_party/dav1d/src/cdef.h --- a/third_party/dav1d/src/cdef.h +++ b/third_party/dav1d/src/cdef.h @@ -50,11 +50,11 @@ // present (according to $edges), then the pre-filter data is located in // $dst. However, the edge pixels above $dst may be post-filter, so in // order to get access to pre-filter top pixels, use $top. #define decl_cdef_fn(name) \ void (name)(pixel *dst, ptrdiff_t stride, const_left_pixel_row_2px left, \ - /*const*/ pixel *const top[2], int pri_strength, int sec_strength, \ + const pixel *top, int pri_strength, int sec_strength, \ int dir, int damping, enum CdefEdgeFlags edges HIGHBD_DECL_SUFFIX) typedef decl_cdef_fn(*cdef_fn); #define decl_cdef_dir_fn(name) \ int (name)(const pixel *dst, ptrdiff_t dst_stride, unsigned *var HIGHBD_DECL_SUFFIX) diff --git a/third_party/dav1d/src/cdef_apply_tmpl.c b/third_party/dav1d/src/cdef_apply_tmpl.c --- a/third_party/dav1d/src/cdef_apply_tmpl.c +++ b/third_party/dav1d/src/cdef_apply_tmpl.c @@ -37,28 +37,32 @@ enum Backup2x8Flags { BACKUP_2X8_Y = 1 << 0, BACKUP_2X8_UV = 1 << 1, }; -static void backup2lines(pixel *const dst[3][2], - /*const*/ pixel *const src[3], - const ptrdiff_t src_stride[2], int y_off, int w, +static void backup2lines(pixel *const dst[3], /*const*/ pixel *const src[3], + const ptrdiff_t stride[2], const enum Dav1dPixelLayout layout) { - pixel_copy(dst[0][0], src[0] + (y_off - 2) * PXSTRIDE(src_stride[0]), w); - pixel_copy(dst[0][1], src[0] + (y_off - 1) * PXSTRIDE(src_stride[0]), w); - - if (layout == DAV1D_PIXEL_LAYOUT_I400) return; - const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; - const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + const ptrdiff_t y_stride = PXSTRIDE(stride[0]); + if (y_stride < 0) + pixel_copy(dst[0] + y_stride, src[0] + 7 * y_stride, -2 * y_stride); + else + pixel_copy(dst[0], src[0] + 6 * y_stride, 2 * y_stride); - w >>= ss_hor; - y_off >>= ss_ver; - pixel_copy(dst[1][0], src[1] + (y_off - 2) * PXSTRIDE(src_stride[1]), w); - pixel_copy(dst[1][1], src[1] + (y_off - 1) * PXSTRIDE(src_stride[1]), w); - pixel_copy(dst[2][0], src[2] + (y_off - 2) * PXSTRIDE(src_stride[1]), w); - pixel_copy(dst[2][1], src[2] + (y_off - 1) * PXSTRIDE(src_stride[1]), w); + if (layout != DAV1D_PIXEL_LAYOUT_I400) { + const ptrdiff_t uv_stride = PXSTRIDE(stride[1]); + if (uv_stride < 0) { + const int uv_off = layout == DAV1D_PIXEL_LAYOUT_I420 ? 3 : 7; + pixel_copy(dst[1] + uv_stride, src[1] + uv_off * uv_stride, -2 * uv_stride); + pixel_copy(dst[2] + uv_stride, src[2] + uv_off * uv_stride, -2 * uv_stride); + } else { + const int uv_off = layout == DAV1D_PIXEL_LAYOUT_I420 ? 2 : 6; + pixel_copy(dst[1], src[1] + uv_off * uv_stride, 2 * uv_stride); + pixel_copy(dst[2], src[2] + uv_off * uv_stride, 2 * uv_stride); + } + } } static void backup2x8(pixel dst[3][8][2], /*const*/ pixel *const src[3], const ptrdiff_t src_stride[2], int x_off, @@ -103,32 +107,28 @@ const int sbsz = 16; const int sb64w = f->sb128w << 1; const int damping = f->frame_hdr->cdef.damping + bitdepth_min_8; const enum Dav1dPixelLayout layout = f->cur.p.layout; const int uv_idx = DAV1D_PIXEL_LAYOUT_I444 - layout; - const int has_chroma = layout != DAV1D_PIXEL_LAYOUT_I400; const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; for (int bit = 0, by = by_start; by < by_end; by += 2, edges |= CDEF_HAVE_TOP) { const int tf = f->lf.top_pre_cdef_toggle; const int by_idx = by & 30; if (by + 2 >= f->bh) edges &= ~CDEF_HAVE_BOTTOM; - if (edges & CDEF_HAVE_BOTTOM) { - // backup pre-filter data for next iteration - backup2lines(f->lf.cdef_line[!tf], ptrs, f->cur.stride, - 8, f->bw * 4, layout); - } + if (edges & CDEF_HAVE_BOTTOM) // backup pre-filter data for next iteration + backup2lines(f->lf.cdef_line[!tf], ptrs, f->cur.stride, layout); - pixel lr_bak[2 /* idx */][3 /* plane */][8 /* y */][2 /* x */]; + ALIGN_STK_16(pixel, lr_bak, 2 /* idx */, [3 /* plane */][8 /* y */][2 /* x */]); pixel *iptrs[3] = { ptrs[0], ptrs[1], ptrs[2] }; edges &= ~CDEF_HAVE_LEFT; edges |= CDEF_HAVE_RIGHT; enum Backup2x8Flags prev_flag = 0; for (int sbx = 0, last_skip = 1; sbx < sb64w; sbx++, edges |= CDEF_HAVE_LEFT) { - const int sb128x = sbx >>1; + const int sb128x = sbx >> 1; const int sb64_idx = ((by & sbsz) >> 3) + (sbx & 1); const int cdef_idx = lflvl[sb128x].cdef_idx[sb64_idx]; if (cdef_idx == -1 || (!f->frame_hdr->cdef.y_strength[cdef_idx] && !f->frame_hdr->cdef.uv_strength[cdef_idx])) @@ -139,10 +139,20 @@ const int y_lvl = f->frame_hdr->cdef.y_strength[cdef_idx]; const int uv_lvl = f->frame_hdr->cdef.uv_strength[cdef_idx]; const enum Backup2x8Flags flag = !!y_lvl + (!!uv_lvl << 1); + const int y_pri_lvl = (y_lvl >> 2) << bitdepth_min_8; + int y_sec_lvl = y_lvl & 3; + y_sec_lvl += y_sec_lvl == 3; + y_sec_lvl <<= bitdepth_min_8; + + const int uv_pri_lvl = (uv_lvl >> 2) << bitdepth_min_8; + int uv_sec_lvl = uv_lvl & 3; + uv_sec_lvl += uv_sec_lvl == 3; + uv_sec_lvl <<= bitdepth_min_8; + pixel *bptrs[3] = { iptrs[0], iptrs[1], iptrs[2] }; for (int bx = sbx * sbsz; bx < imin((sbx + 1) * sbsz, f->bw); bx += 2, edges |= CDEF_HAVE_LEFT) { if (bx + 2 >= f->bw) edges &= ~CDEF_HAVE_RIGHT; @@ -167,45 +177,36 @@ if (edges & CDEF_HAVE_RIGHT) { // backup pre-filter data for next iteration backup2x8(lr_bak[!bit], bptrs, f->cur.stride, 8, layout, flag); } - // the actual filter - const int y_pri_lvl = (y_lvl >> 2) << bitdepth_min_8; - int y_sec_lvl = y_lvl & 3; - y_sec_lvl += y_sec_lvl == 3; - y_sec_lvl <<= bitdepth_min_8; - const int uv_pri_lvl = (uv_lvl >> 2) << bitdepth_min_8; - int uv_sec_lvl = uv_lvl & 3; - uv_sec_lvl += uv_sec_lvl == 3; - uv_sec_lvl <<= bitdepth_min_8; + int dir; unsigned variance; - const int dir = dsp->cdef.dir(bptrs[0], f->cur.stride[0], - &variance HIGHBD_CALL_SUFFIX); - if (y_lvl) { + if (y_pri_lvl || uv_pri_lvl) + dir = dsp->cdef.dir(bptrs[0], f->cur.stride[0], + &variance HIGHBD_CALL_SUFFIX); + + if (y_pri_lvl) { + const int adj_y_pri_lvl = adjust_strength(y_pri_lvl, variance); + if (adj_y_pri_lvl || y_sec_lvl) + dsp->cdef.fb[0](bptrs[0], f->cur.stride[0], lr_bak[bit][0], + &f->lf.cdef_line[tf][0][bx * 4], + adj_y_pri_lvl, y_sec_lvl, dir, + damping, edges HIGHBD_CALL_SUFFIX); + } else if (y_sec_lvl) dsp->cdef.fb[0](bptrs[0], f->cur.stride[0], lr_bak[bit][0], - (pixel *const [2]) { - &f->lf.cdef_line[tf][0][0][bx * 4], - &f->lf.cdef_line[tf][0][1][bx * 4], - }, - adjust_strength(y_pri_lvl, variance), - y_sec_lvl, y_pri_lvl ? dir : 0, + &f->lf.cdef_line[tf][0][bx * 4], + 0, y_sec_lvl, 0, damping, edges HIGHBD_CALL_SUFFIX); - } - if (uv_lvl && has_chroma) { - const int uvdir = - f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I422 ? dir : - ((uint8_t[]) { 7, 0, 2, 4, 5, 6, 6, 6 })[dir]; + if (uv_lvl) { + assert(layout != DAV1D_PIXEL_LAYOUT_I400); + const int uvdir = uv_pri_lvl ? layout == DAV1D_PIXEL_LAYOUT_I422 ? + ((const uint8_t[]) { 7, 0, 2, 4, 5, 6, 6, 6 })[dir] : dir : 0; for (int pl = 1; pl <= 2; pl++) { - dsp->cdef.fb[uv_idx](bptrs[pl], f->cur.stride[1], - lr_bak[bit][pl], - (pixel *const [2]) { - &f->lf.cdef_line[tf][pl][0][bx * 4 >> ss_hor], - &f->lf.cdef_line[tf][pl][1][bx * 4 >> ss_hor], - }, - uv_pri_lvl, uv_sec_lvl, - uv_pri_lvl ? uvdir : 0, + dsp->cdef.fb[uv_idx](bptrs[pl], f->cur.stride[1], lr_bak[bit][pl], + &f->lf.cdef_line[tf][pl][bx * 4 >> ss_hor], + uv_pri_lvl, uv_sec_lvl, uvdir, damping - 1, edges HIGHBD_CALL_SUFFIX); } } bit ^= 1; diff --git a/third_party/dav1d/src/cdef_tmpl.c b/third_party/dav1d/src/cdef_tmpl.c --- a/third_party/dav1d/src/cdef_tmpl.c +++ b/third_party/dav1d/src/cdef_tmpl.c @@ -30,33 +30,34 @@ #include #include "common/intops.h" #include "src/cdef.h" +#include "src/tables.h" static inline int constrain(const int diff, const int threshold, - const int damping) + const int shift) { - if (!threshold) return 0; - const int shift = imax(0, damping - ulog2(threshold)); - return apply_sign(imin(abs(diff), imax(0, threshold - (abs(diff) >> shift))), - diff); + const int adiff = abs(diff); + return apply_sign(imin(adiff, imax(0, threshold - (adiff >> shift))), diff); } -static inline void fill(uint16_t *tmp, const ptrdiff_t stride, +static inline void fill(int16_t *tmp, const ptrdiff_t stride, const int w, const int h) { + /* Use a value that's a large positive number when interpreted as unsigned, + * and a large negative number when interpreted as signed. */ for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) - tmp[x] = INT16_MAX; + tmp[x] = INT16_MIN; tmp += stride; } } -static void padding(uint16_t *tmp, const ptrdiff_t tmp_stride, +static void padding(int16_t *tmp, const ptrdiff_t tmp_stride, const pixel *src, const ptrdiff_t src_stride, - const pixel (*left)[2], pixel *const top[2], + const pixel (*left)[2], const pixel *top, const int w, const int h, const enum CdefEdgeFlags edges) { // fill extended input buffer int x_start = -2, x_end = w + 2, y_start = -2, y_end = h + 2; @@ -75,13 +76,15 @@ if (!(edges & CDEF_HAVE_RIGHT)) { fill(tmp + y_start * tmp_stride + w, tmp_stride, 2, y_end - y_start); x_end -= 2; } - for (int y = y_start; y < 0; y++) + for (int y = y_start; y < 0; y++) { for (int x = x_start; x < x_end; x++) - tmp[x + y * tmp_stride] = top[y & 1][x]; + tmp[x + y * tmp_stride] = top[x]; + top += PXSTRIDE(src_stride); + } for (int y = 0; y < h; y++) for (int x = x_start; x < 0; x++) tmp[x + y * tmp_stride] = left[y][2 + x]; for (int y = 0; y < y_end; y++) { for (int x = (y < h) ? 0 : x_start; x < x_end; x++) @@ -91,96 +94,134 @@ } } static NOINLINE void cdef_filter_block_c(pixel *dst, const ptrdiff_t dst_stride, - const pixel (*left)[2], /*const*/ pixel *const top[2], - const int w, const int h, const int pri_strength, - const int sec_strength, const int dir, - const int damping, const enum CdefEdgeFlags edges - HIGHBD_DECL_SUFFIX) + const pixel (*left)[2], const pixel *const top, + const int pri_strength, const int sec_strength, + const int dir, const int damping, const int w, int h, + const enum CdefEdgeFlags edges HIGHBD_DECL_SUFFIX) { - static const int8_t cdef_directions[8 /* dir */][2 /* pass */] = { - { -1 * 12 + 1, -2 * 12 + 2 }, - { 0 * 12 + 1, -1 * 12 + 2 }, - { 0 * 12 + 1, 0 * 12 + 2 }, - { 0 * 12 + 1, 1 * 12 + 2 }, - { 1 * 12 + 1, 2 * 12 + 2 }, - { 1 * 12 + 0, 2 * 12 + 1 }, - { 1 * 12 + 0, 2 * 12 + 0 }, - { 1 * 12 + 0, 2 * 12 - 1 } - }; const ptrdiff_t tmp_stride = 12; assert((w == 4 || w == 8) && (h == 4 || h == 8)); - uint16_t tmp_buf[144]; // 12*12 is the maximum value of tmp_stride * (h + 4) - uint16_t *tmp = tmp_buf + 2 * tmp_stride + 2; - const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; - const int pri_tap = 4 - ((pri_strength >> bitdepth_min_8) & 1); + int16_t tmp_buf[144]; // 12*12 is the maximum value of tmp_stride * (h + 4) + int16_t *tmp = tmp_buf + 2 * tmp_stride + 2; padding(tmp, tmp_stride, dst, dst_stride, left, top, w, h, edges); - // run actual filter - for (int y = 0; y < h; y++) { - for (int x = 0; x < w; x++) { - int sum = 0; - const int px = dst[x]; - int max = px, min = px; - int pri_tap_k = pri_tap; - for (int k = 0; k < 2; k++) { - const int off1 = cdef_directions[dir][k]; - const int p0 = tmp[x + off1]; - const int p1 = tmp[x - off1]; - sum += pri_tap_k * constrain(p0 - px, pri_strength, damping); - sum += pri_tap_k * constrain(p1 - px, pri_strength, damping); - // if pri_tap_k == 4 then it becomes 2 else it remains 3 - pri_tap_k -= (pri_tap_k << 1) - 6; - if (p0 != INT16_MAX) max = imax(p0, max); - if (p1 != INT16_MAX) max = imax(p1, max); - min = imin(p0, min); - min = imin(p1, min); - const int off2 = cdef_directions[(dir + 2) & 7][k]; - const int s0 = tmp[x + off2]; - const int s1 = tmp[x - off2]; - const int off3 = cdef_directions[(dir + 6) & 7][k]; - const int s2 = tmp[x + off3]; - const int s3 = tmp[x - off3]; - if (s0 != INT16_MAX) max = imax(s0, max); - if (s1 != INT16_MAX) max = imax(s1, max); - if (s2 != INT16_MAX) max = imax(s2, max); - if (s3 != INT16_MAX) max = imax(s3, max); - min = imin(s0, min); - min = imin(s1, min); - min = imin(s2, min); - min = imin(s3, min); - // sec_tap starts at 2 and becomes 1 - const int sec_tap = 2 - k; - sum += sec_tap * constrain(s0 - px, sec_strength, damping); - sum += sec_tap * constrain(s1 - px, sec_strength, damping); - sum += sec_tap * constrain(s2 - px, sec_strength, damping); - sum += sec_tap * constrain(s3 - px, sec_strength, damping); + if (pri_strength) { + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + const int pri_tap = 4 - ((pri_strength >> bitdepth_min_8) & 1); + const int pri_shift = imax(0, damping - ulog2(pri_strength)); + if (sec_strength) { + const int sec_shift = imax(0, damping - ulog2(sec_strength)); + do { + for (int x = 0; x < w; x++) { + const int px = dst[x]; + int sum = 0; + int max = px, min = px; + int pri_tap_k = pri_tap; + for (int k = 0; k < 2; k++) { + const int off1 = dav1d_cdef_directions[dir + 2][k]; // dir + const int p0 = tmp[x + off1]; + const int p1 = tmp[x - off1]; + sum += pri_tap_k * constrain(p0 - px, pri_strength, pri_shift); + sum += pri_tap_k * constrain(p1 - px, pri_strength, pri_shift); + // if pri_tap_k == 4 then it becomes 2 else it remains 3 + pri_tap_k = (pri_tap_k & 3) | 2; + min = umin(p0, min); + max = imax(p0, max); + min = umin(p1, min); + max = imax(p1, max); + const int off2 = dav1d_cdef_directions[dir + 4][k]; // dir + 2 + const int off3 = dav1d_cdef_directions[dir + 0][k]; // dir - 2 + const int s0 = tmp[x + off2]; + const int s1 = tmp[x - off2]; + const int s2 = tmp[x + off3]; + const int s3 = tmp[x - off3]; + // sec_tap starts at 2 and becomes 1 + const int sec_tap = 2 - k; + sum += sec_tap * constrain(s0 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s1 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s2 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s3 - px, sec_strength, sec_shift); + min = umin(s0, min); + max = imax(s0, max); + min = umin(s1, min); + max = imax(s1, max); + min = umin(s2, min); + max = imax(s2, max); + min = umin(s3, min); + max = imax(s3, max); + } + dst[x] = iclip(px + ((sum - (sum < 0) + 8) >> 4), min, max); + } + dst += PXSTRIDE(dst_stride); + tmp += tmp_stride; + } while (--h); + } else { // pri_strength only + do { + for (int x = 0; x < w; x++) { + const int px = dst[x]; + int sum = 0; + int pri_tap_k = pri_tap; + for (int k = 0; k < 2; k++) { + const int off = dav1d_cdef_directions[dir + 2][k]; // dir + const int p0 = tmp[x + off]; + const int p1 = tmp[x - off]; + sum += pri_tap_k * constrain(p0 - px, pri_strength, pri_shift); + sum += pri_tap_k * constrain(p1 - px, pri_strength, pri_shift); + pri_tap_k = (pri_tap_k & 3) | 2; + } + dst[x] = px + ((sum - (sum < 0) + 8) >> 4); + } + dst += PXSTRIDE(dst_stride); + tmp += tmp_stride; + } while (--h); + } + } else { // sec_strength only + assert(sec_strength); + const int sec_shift = imax(0, damping - ulog2(sec_strength)); + do { + for (int x = 0; x < w; x++) { + const int px = dst[x]; + int sum = 0; + for (int k = 0; k < 2; k++) { + const int off1 = dav1d_cdef_directions[dir + 4][k]; // dir + 2 + const int off2 = dav1d_cdef_directions[dir + 0][k]; // dir - 2 + const int s0 = tmp[x + off1]; + const int s1 = tmp[x - off1]; + const int s2 = tmp[x + off2]; + const int s3 = tmp[x - off2]; + const int sec_tap = 2 - k; + sum += sec_tap * constrain(s0 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s1 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s2 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s3 - px, sec_strength, sec_shift); + } + dst[x] = px + ((sum - (sum < 0) + 8) >> 4); } - dst[x] = iclip(px + ((8 + sum - (sum < 0)) >> 4), min, max); - } - dst += PXSTRIDE(dst_stride); - tmp += tmp_stride; + dst += PXSTRIDE(dst_stride); + tmp += tmp_stride; + } while (--h); } } #define cdef_fn(w, h) \ static void cdef_filter_block_##w##x##h##_c(pixel *const dst, \ const ptrdiff_t stride, \ const pixel (*left)[2], \ - /*const*/ pixel *const top[2], \ + const pixel *const top, \ const int pri_strength, \ const int sec_strength, \ const int dir, \ const int damping, \ const enum CdefEdgeFlags edges \ HIGHBD_DECL_SUFFIX) \ { \ - cdef_filter_block_c(dst, stride, left, top, w, h, pri_strength, sec_strength, \ - dir, damping, edges HIGHBD_TAIL_SUFFIX); \ + cdef_filter_block_c(dst, stride, left, top, pri_strength, sec_strength, \ + dir, damping, w, h, edges HIGHBD_TAIL_SUFFIX); \ } cdef_fn(4, 4); cdef_fn(4, 8); cdef_fn(8, 8); diff --git a/third_party/dav1d/src/cpu.h b/third_party/dav1d/src/cpu.h --- a/third_party/dav1d/src/cpu.h +++ b/third_party/dav1d/src/cpu.h @@ -40,9 +40,10 @@ #include "src/ppc/cpu.h" #elif ARCH_X86 #include "src/x86/cpu.h" #endif +void dav1d_init_cpu(void); unsigned dav1d_get_cpu_flags(void); -DAV1D_API void dav1d_set_cpu_flags_mask(const unsigned mask); +DAV1D_API void dav1d_set_cpu_flags_mask(unsigned mask); #endif /* DAV1D_SRC_CPU_H */ diff --git a/third_party/dav1d/src/cpu.c b/third_party/dav1d/src/cpu.c --- a/third_party/dav1d/src/cpu.c +++ b/third_party/dav1d/src/cpu.c @@ -28,28 +28,31 @@ #include #include "src/cpu.h" +static unsigned flags = 0; +#if ARCH_X86 +/* Disable AVX-512 by default for the time being */ +static unsigned flags_mask = ~DAV1D_X86_CPU_FLAG_AVX512ICL; +#else static unsigned flags_mask = -1; +#endif + +COLD void dav1d_init_cpu(void) { +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + flags = dav1d_get_cpu_flags_arm(); +#elif ARCH_PPC64LE + flags = dav1d_get_cpu_flags_ppc(); +#elif ARCH_X86 + flags = dav1d_get_cpu_flags_x86(); +#endif +#endif +} COLD unsigned dav1d_get_cpu_flags(void) { - static unsigned flags; - static uint8_t checked = 0; - - if (!checked) { -#if (ARCH_AARCH64 || ARCH_ARM) && HAVE_ASM - flags = dav1d_get_cpu_flags_arm(); -#elif ARCH_PPC64LE && HAVE_ASM - flags = dav1d_get_cpu_flags_ppc(); -#elif ARCH_X86 && HAVE_ASM - flags = dav1d_get_cpu_flags_x86(); -#else - flags = 0; -#endif - checked = 1; - } return flags & flags_mask; } COLD void dav1d_set_cpu_flags_mask(const unsigned mask) { flags_mask = mask; diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -221,24 +221,22 @@ const int bw4, const int bh4, const int w4, const int h4, const int have_left, const int have_top, const int ref, uint64_t masks[2]) { - const Dav1dFrameContext *const f = t->f; - const ptrdiff_t b4_stride = f->b4_stride; - const refmvs *const r = &f->mvs[t->by * b4_stride + t->bx]; + /*const*/ refmvs_block *const *r = &t->rt.r[(t->by & 31) + 5]; int count = 0; int have_topleft = have_top && have_left; int have_topright = imax(bw4, bh4) < 32 && have_top && t->bx + bw4 < t->ts->tiling.col_end && (intra_edge_flags & EDGE_I444_TOP_HAS_RIGHT); -#define bs(rp) dav1d_block_dimensions[dav1d_sbtype_to_bs[(rp)->sb_type]] -#define matches(rp) ((rp)->ref[0] == ref + 1 && (rp)->ref[1] == -1) +#define bs(rp) dav1d_block_dimensions[(rp)->bs] +#define matches(rp) ((rp)->ref.ref[0] == ref + 1 && (rp)->ref.ref[1] == -1) if (have_top) { - const refmvs *r2 = &r[-b4_stride]; + const refmvs_block *r2 = &r[-1][t->bx]; if (matches(r2)) { masks[0] |= 1; count = 1; } int aw4 = bs(r2)[0]; @@ -258,84 +256,82 @@ mask <<= aw4; } } } if (have_left) { - const refmvs *r2 = &r[-1]; - if (matches(r2)) { + /*const*/ refmvs_block *const *r2 = r; + if (matches(&r2[0][t->bx - 1])) { masks[1] |= 1; if (++count >= 8) return; } - int lh4 = bs(r2)[1]; + int lh4 = bs(&r2[0][t->bx - 1])[1]; if (lh4 >= bh4) { if (t->by & (lh4 - 1)) have_topleft = 0; } else { unsigned mask = 1 << lh4; for (int y = lh4; y < h4; y += lh4) { - r2 += lh4 * b4_stride; - if (matches(r2)) { + r2 += lh4; + if (matches(&r2[0][t->bx - 1])) { masks[1] |= mask; if (++count >= 8) return; } - lh4 = bs(r2)[1]; + lh4 = bs(&r2[0][t->bx - 1])[1]; mask <<= lh4; } } } - if (have_topleft && matches(&r[-(1 + b4_stride)])) { + if (have_topleft && matches(&r[-1][t->bx - 1])) { masks[1] |= 1ULL << 32; if (++count >= 8) return; } - if (have_topright && matches(&r[bw4 - b4_stride])) { + if (have_topright && matches(&r[-1][t->bx + bw4])) { masks[0] |= 1ULL << 32; } #undef matches } static void derive_warpmv(const Dav1dTileContext *const t, const int bw4, const int bh4, - const uint64_t masks[2], const struct mv mv, + const uint64_t masks[2], const union mv mv, Dav1dWarpedMotionParams *const wmp) { int pts[8][2 /* in, out */][2 /* x, y */], np = 0; - const Dav1dFrameContext *const f = t->f; - const ptrdiff_t b4_stride = f->b4_stride; - const refmvs *const r = &f->mvs[t->by * b4_stride + t->bx]; + /*const*/ refmvs_block *const *r = &t->rt.r[(t->by & 31) + 5]; #define add_sample(dx, dy, sx, sy, rp) do { \ pts[np][0][0] = 16 * (2 * dx + sx * bs(rp)[0]) - 8; \ pts[np][0][1] = 16 * (2 * dy + sy * bs(rp)[1]) - 8; \ - pts[np][1][0] = pts[np][0][0] + (rp)->mv[0].x; \ - pts[np][1][1] = pts[np][0][1] + (rp)->mv[0].y; \ + pts[np][1][0] = pts[np][0][0] + (rp)->mv.mv[0].x; \ + pts[np][1][1] = pts[np][0][1] + (rp)->mv.mv[0].y; \ np++; \ } while (0) // use masks[] to find the projectable motion vectors in the edges if ((unsigned) masks[0] == 1 && !(masks[1] >> 32)) { - const int off = t->bx & (bs(&r[-b4_stride])[0] - 1); - add_sample(-off, 0, 1, -1, &r[-b4_stride]); + const int off = t->bx & (bs(&r[-1][t->bx])[0] - 1); + add_sample(-off, 0, 1, -1, &r[-1][t->bx]); } else for (unsigned off = 0, xmask = (uint32_t) masks[0]; np < 8 && xmask;) { // top const int tz = ctz(xmask); off += tz; xmask >>= tz; - add_sample(off, 0, 1, -1, &r[off - b4_stride]); + add_sample(off, 0, 1, -1, &r[-1][t->bx + off]); xmask &= ~1; } if (np < 8 && masks[1] == 1) { - const int off = t->by & (bs(&r[-1])[1] - 1); - add_sample(0, -off, -1, 1, &r[-1 - off * b4_stride]); + const int off = t->by & (bs(&r[0][t->bx - 1])[1] - 1); + add_sample(0, -off, -1, 1, &r[-off][t->bx - 1]); } else for (unsigned off = 0, ymask = (uint32_t) masks[1]; np < 8 && ymask;) { // left const int tz = ctz(ymask); off += tz; ymask >>= tz; - add_sample(0, off, -1, 1, &r[off * b4_stride - 1]); + add_sample(0, off, -1, 1, &r[off][t->bx - 1]); ymask &= ~1; } if (np < 8 && masks[1] >> 32) // top/left - add_sample(0, 0, -1, -1, &r[-(1 + b4_stride)]); + add_sample(0, 0, -1, -1, &r[-1][t->bx - 1]); if (np < 8 && masks[0] >> 32) // top/right - add_sample(bw4, 0, 1, -1, &r[bw4 - b4_stride]); + add_sample(bw4, 0, 1, -1, &r[-1][t->bx + bw4]); assert(np > 0 && np <= 8); #undef bs // select according to motion vector difference against a threshold int mvd[8], ret = 0; @@ -623,14 +619,14 @@ const Dav1dFrameContext *const f = t->f; const uint8_t *const b_dim = dav1d_block_dimensions[bs]; const int bw4 = b_dim[0], bh4 = b_dim[1]; // var-tx tree coding - b->tx_split[0] = b->tx_split[1] = 0; + uint16_t tx_split[2] = { 0 }; b->max_ytx = dav1d_max_txfm_size_for_bs[bs][0]; - if (f->frame_hdr->segmentation.lossless[b->seg_id] || - b->max_ytx == TX_4X4) + if (!b->skip && (f->frame_hdr->segmentation.lossless[b->seg_id] || + b->max_ytx == TX_4X4)) { b->max_ytx = b->uvtx = TX_4X4; if (f->frame_hdr->txfm_mode == DAV1D_TX_SWITCHABLE) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir tx, off, TX_4X4) @@ -643,33 +639,34 @@ #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir tx, off, mul * b_dim[2 + diridx]) case_set(bh4, l., 1, by4); case_set(bw4, a->, 0, bx4); #undef set_ctx - } else { - assert(f->frame_hdr->txfm_mode == DAV1D_TX_LARGEST); } b->uvtx = dav1d_max_txfm_size_for_bs[bs][f->cur.p.layout]; } else { assert(bw4 <= 16 || bh4 <= 16 || b->max_ytx == TX_64X64); int y, x, y_off, x_off; const TxfmInfo *const ytx = &dav1d_txfm_dimensions[b->max_ytx]; for (y = 0, y_off = 0; y < bh4; y += ytx->h, y_off++) { for (x = 0, x_off = 0; x < bw4; x += ytx->w, x_off++) { - read_tx_tree(t, b->max_ytx, 0, b->tx_split, x_off, y_off); + read_tx_tree(t, b->max_ytx, 0, tx_split, x_off, y_off); // contexts are updated inside read_tx_tree() t->bx += ytx->w; } t->bx -= x; t->by += ytx->h; } t->by -= y; if (DEBUG_BLOCK_INFO) printf("Post-vartxtree[%x/%x]: r=%d\n", - b->tx_split[0], b->tx_split[1], t->ts->msac.rng); + tx_split[0], tx_split[1], t->ts->msac.rng); b->uvtx = dav1d_max_txfm_size_for_bs[bs][f->cur.p.layout]; } + assert(!(tx_split[0] & ~0x33)); + b->tx_split0 = (uint8_t)tx_split[0]; + b->tx_split1 = tx_split[1]; } static inline unsigned get_prev_frame_segid(const Dav1dFrameContext *const f, const int by, const int bx, const int w4, int h4, @@ -729,10 +726,22 @@ rep_macro(type, t->dir mode, off, mul * y_mode_nofilt); \ rep_macro(type, t->dir intra, off, mul) case_set(bh4, l., 1, by4); case_set(bw4, a->, 0, bx4); #undef set_ctx + if (f->frame_hdr->frame_type & 1) { + refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; + for (int x = 0; x < bw4; x++) { + r[x].ref.ref[0] = 0; + r[x].bs = bs; + } + refmvs_block *const *rr = &t->rt.r[(t->by & 31) + 5]; + for (int y = 0; y < bh4 - 1; y++) { + rr[y][t->bx + bw4 - 1].ref.ref[0] = 0; + rr[y][t->bx + bw4 - 1].bs = bs; + } + } if (has_chroma) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir uvmode, off, mul * b->uv_mode) case_set(cbh4, l., 1, cby4); @@ -741,14 +750,38 @@ } } else { if (f->frame_hdr->frame_type & 1 /* not intrabc */ && b->comp_type == COMP_INTER_NONE && b->motion_mode == MM_WARP) { - uint64_t mask[2] = { 0, 0 }; - find_matching_ref(t, intra_edge_flags, bw4, bh4, w4, h4, - have_left, have_top, b->ref[0], mask); - derive_warpmv(t, bw4, bh4, mask, b->mv[0], &t->warpmv); + if (b->matrix[0] == SHRT_MIN) { + t->warpmv.type = DAV1D_WM_TYPE_IDENTITY; + } else { + t->warpmv.type = DAV1D_WM_TYPE_AFFINE; + t->warpmv.matrix[2] = b->matrix[0] + 0x10000; + t->warpmv.matrix[3] = b->matrix[1]; + t->warpmv.matrix[4] = b->matrix[2]; + t->warpmv.matrix[5] = b->matrix[3] + 0x10000; + dav1d_set_affine_mv2d(bw4, bh4, b->mv2d, &t->warpmv, + t->bx, t->by); + dav1d_get_shear_params(&t->warpmv); +#define signabs(v) v < 0 ? '-' : ' ', abs(v) + if (DEBUG_BLOCK_INFO) + printf("[ %c%x %c%x %c%x\n %c%x %c%x %c%x ]\n" + "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x, mv=y:%d,x:%d\n", + signabs(t->warpmv.matrix[0]), + signabs(t->warpmv.matrix[1]), + signabs(t->warpmv.matrix[2]), + signabs(t->warpmv.matrix[3]), + signabs(t->warpmv.matrix[4]), + signabs(t->warpmv.matrix[5]), + signabs(t->warpmv.alpha), + signabs(t->warpmv.beta), + signabs(t->warpmv.gamma), + signabs(t->warpmv.delta), + b->mv2d.y, b->mv2d.x); +#undef signabs + } } if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; const uint8_t *const filter = dav1d_filter_dir[b->filter2d]; #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ @@ -757,10 +790,25 @@ rep_macro(type, t->dir intra, off, 0) case_set(bh4, l., 1, by4); case_set(bw4, a->, 0, bx4); #undef set_ctx + if (f->frame_hdr->frame_type & 1) { + refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; + for (int x = 0; x < bw4; x++) { + r[x].ref.ref[0] = b->ref[0] + 1; + r[x].mv.mv[0] = b->mv[0]; + r[x].bs = bs; + } + refmvs_block *const *rr = &t->rt.r[(t->by & 31) + 5]; + for (int y = 0; y < bh4 - 1; y++) { + rr[y][t->bx + bw4 - 1].ref.ref[0] = b->ref[0] + 1; + rr[y][t->bx + bw4 - 1].mv.mv[0] = b->mv[0]; + rr[y][t->bx + bw4 - 1].bs = bs; + } + } + if (has_chroma) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir uvmode, off, mul * DC_PRED) case_set(cbh4, l., 1, cby4); case_set(cbw4, a->, 0, cbx4); @@ -1245,39 +1293,35 @@ memcpy(t->al_pal[1][by4 + y][pl], pal[pl], 16); } } } if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { - splat_intraref(f->mvs, f->b4_stride, t->by, t->bx, bs, - y_mode_nofilt); + splat_intraref(&t->rt, t->by, t->bx, bs); } } else if (!(f->frame_hdr->frame_type & 1)) { // intra block copy - candidate_mv mvstack[8]; - int n_mvs; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, NULL, - (int[2]) { -1, -1 }, f->bw, f->bh, - bs, bp, t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + refmvs_candidate mvstack[8]; + int n_mvs, ctx; + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { 0, -1 }}, + bs, intra_edge_flags, t->by, t->bx); - if (mvlist[0][0].y | mvlist[0][0].x) - b->mv[0] = mvlist[0][0]; - else if (mvlist[0][1].y | mvlist[0][1].x) - b->mv[0] = mvlist[0][1]; + if (mvstack[0].mv.mv[0].n) + b->mv[0] = mvstack[0].mv.mv[0]; + else if (mvstack[1].mv.mv[0].n) + b->mv[0] = mvstack[1].mv.mv[0]; else { if (t->by - (16 << f->seq_hdr->sb128) < ts->tiling.row_start) { b->mv[0].y = 0; b->mv[0].x = -(512 << f->seq_hdr->sb128) - 2048; } else { b->mv[0].y = -(512 << f->seq_hdr->sb128); b->mv[0].x = 0; } } - const struct mv ref = b->mv[0]; + const union mv ref = b->mv[0]; read_mv_residual(t, &b->mv[0], &ts->cdf.dmv, 0); // clip intrabc motion vector to decoded parts of current tile int border_left = ts->tiling.col_start * 4; int border_top = ts->tiling.row_start * 4; @@ -1335,22 +1379,22 @@ b->mv[0].y = (src_top - t->by * 4) * 8; if (DEBUG_BLOCK_INFO) printf("Post-dmv[%d/%d,ref=%d/%d|%d/%d]: r=%d\n", b->mv[0].y, b->mv[0].x, ref.y, ref.x, - mvlist[0][0].y, mvlist[0][0].x, ts->msac.rng); + mvstack[0].mv.mv[0].y, mvstack[0].mv.mv[0].x, ts->msac.rng); read_vartx_tree(t, b, bs, bx4, by4); // reconstruction if (f->frame_thread.pass == 1) { f->bd_fn.read_coef_blocks(t, bs, b); b->filter2d = FILTER_2D_BILINEAR; } else { if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; } - splat_intrabc_mv(f->mvs, f->b4_stride, t->by, t->bx, bs, b->mv[0]); + splat_intrabc_mv(&t->rt, t->by, t->bx, bs, b->mv[0]); #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir tx_intra, off, mul * b_dim[2 + diridx]); \ rep_macro(type, t->dir mode, off, mul * DC_PRED); \ rep_macro(type, t->dir pal_sz, off, 0); \ @@ -1395,21 +1439,19 @@ b->comp_type = COMP_INTER_AVG; b->inter_mode = NEARESTMV_NEARESTMV; b->drl_idx = NEAREST_DRL; has_subpel_filter = 0; - candidate_mv mvstack[8]; + refmvs_candidate mvstack[8]; int n_mvs, ctx; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, &ctx, - (int[2]) { b->ref[0], b->ref[1] }, f->bw, f->bh, - bs, bp, t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { + b->ref[0] + 1, b->ref[1] + 1 }}, + bs, intra_edge_flags, t->by, t->bx); - b->mv[0] = mvstack[0].this_mv; - b->mv[1] = mvstack[0].comp_mv; + b->mv[0] = mvstack[0].mv.mv[0]; + b->mv[1] = mvstack[0].mv.mv[1]; fix_mv_precision(f->frame_hdr, &b->mv[0]); fix_mv_precision(f->frame_hdr, &b->mv[1]); if (DEBUG_BLOCK_INFO) printf("Post-skipmodeblock[mv=1:y=%d,x=%d,2:y=%d,x=%d,refs=%d+%d\n", b->mv[0].y, b->mv[0].x, b->mv[1].y, b->mv[1].x, @@ -1475,18 +1517,16 @@ } if (DEBUG_BLOCK_INFO) printf("Post-refs[%d/%d]: r=%d\n", b->ref[0], b->ref[1], ts->msac.rng); - candidate_mv mvstack[8]; + refmvs_candidate mvstack[8]; int n_mvs, ctx; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, &ctx, - (int[2]) { b->ref[0], b->ref[1] }, f->bw, f->bh, - bs, bp, t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { + b->ref[0] + 1, b->ref[1] + 1 }}, + bs, intra_edge_flags, t->by, t->bx); b->inter_mode = dav1d_msac_decode_symbol_adapt8(&ts->msac, ts->cdf.m.comp_inter_mode[ctx], N_COMP_INTER_PRED_MODES - 1); if (DEBUG_BLOCK_INFO) @@ -1525,34 +1565,33 @@ b->drl_idx, n_mvs, ts->msac.rng); } } assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); -#define assign_comp_mv(idx, pfx) \ +#define assign_comp_mv(idx) \ switch (im[idx]) { \ case NEARMV: \ case NEARESTMV: \ - b->mv[idx] = mvstack[b->drl_idx].pfx##_mv; \ + b->mv[idx] = mvstack[b->drl_idx].mv.mv[idx]; \ fix_mv_precision(f->frame_hdr, &b->mv[idx]); \ break; \ case GLOBALMV: \ has_subpel_filter |= \ f->frame_hdr->gmv[b->ref[idx]].type == DAV1D_WM_TYPE_TRANSLATION; \ b->mv[idx] = get_gmv_2d(&f->frame_hdr->gmv[b->ref[idx]], \ t->bx, t->by, bw4, bh4, f->frame_hdr); \ - fix_mv_precision(f->frame_hdr, &b->mv[idx]); \ break; \ case NEWMV: \ - b->mv[idx] = mvstack[b->drl_idx].pfx##_mv; \ + b->mv[idx] = mvstack[b->drl_idx].mv.mv[idx]; \ read_mv_residual(t, &b->mv[idx], &ts->cdf.mv, \ !f->frame_hdr->force_integer_mv); \ break; \ } has_subpel_filter = imin(bw4, bh4) == 1 || b->inter_mode != GLOBALMV_GLOBALMV; - assign_comp_mv(0, this); - assign_comp_mv(1, comp); + assign_comp_mv(0); + assign_comp_mv(1); #undef assign_comp_mv if (DEBUG_BLOCK_INFO) printf("Post-residual_mv[1:y=%d,x=%d,2:y=%d,x=%d]: r=%d\n", b->mv[0].y, b->mv[0].x, b->mv[1].y, b->mv[1].x, ts->msac.rng); @@ -1653,18 +1692,15 @@ if (DEBUG_BLOCK_INFO) printf("Post-ref[%d]: r=%d\n", b->ref[0], ts->msac.rng); } b->ref[1] = -1; - candidate_mv mvstack[8]; + refmvs_candidate mvstack[8]; int n_mvs, ctx; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, &ctx, - (int[2]) { b->ref[0], -1 }, f->bw, f->bh, bs, bp, - t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { b->ref[0] + 1, -1 }}, + bs, intra_edge_flags, t->by, t->bx); // mode parsing and mv derivation from ref_mvs if ((seg && (seg->skip || seg->globalmv)) || dav1d_msac_decode_bool_adapt(&ts->msac, ts->cdf.m.newmv_mode[ctx & 7])) @@ -1674,11 +1710,10 @@ ts->cdf.m.globalmv_mode[(ctx >> 3) & 1])) { b->inter_mode = GLOBALMV; b->mv[0] = get_gmv_2d(&f->frame_hdr->gmv[b->ref[0]], t->bx, t->by, bw4, bh4, f->frame_hdr); - fix_mv_precision(f->frame_hdr, &b->mv[0]); has_subpel_filter = imin(bw4, bh4) == 1 || f->frame_hdr->gmv[b->ref[0]].type == DAV1D_WM_TYPE_TRANSLATION; } else { has_subpel_filter = 1; if (dav1d_msac_decode_bool_adapt(&ts->msac, @@ -1700,16 +1735,13 @@ } else { b->inter_mode = NEARESTMV; b->drl_idx = NEAREST_DRL; } assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); - if (b->drl_idx >= NEAR_DRL) { - b->mv[0] = mvstack[b->drl_idx].this_mv; - } else { - b->mv[0] = mvlist[0][b->drl_idx]; + b->mv[0] = mvstack[b->drl_idx].mv.mv[0]; + if (b->drl_idx < NEAR_DRL) fix_mv_precision(f->frame_hdr, &b->mv[0]); - } } if (DEBUG_BLOCK_INFO) printf("Post-intermode[%d,drl=%d,mv=y:%d,x:%d,n_mvs=%d]: r=%d\n", b->inter_mode, b->drl_idx, b->mv[0].y, b->mv[0].x, n_mvs, @@ -1728,14 +1760,14 @@ ts->cdf.m.drl_bit[drl_ctx_v2]); } } assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); if (n_mvs > 1) { - b->mv[0] = mvstack[b->drl_idx].this_mv; + b->mv[0] = mvstack[b->drl_idx].mv.mv[0]; } else { assert(!b->drl_idx); - b->mv[0] = mvlist[0][0]; + b->mv[0] = mvstack[0].mv.mv[0]; fix_mv_precision(f->frame_hdr, &b->mv[0]); } if (DEBUG_BLOCK_INFO) printf("Post-intermode[%d,drl=%d]: r=%d\n", b->inter_mode, b->drl_idx, ts->msac.rng); @@ -1801,22 +1833,34 @@ has_subpel_filter = 0; derive_warpmv(t, bw4, bh4, mask, b->mv[0], &t->warpmv); #define signabs(v) v < 0 ? '-' : ' ', abs(v) if (DEBUG_BLOCK_INFO) printf("[ %c%x %c%x %c%x\n %c%x %c%x %c%x ]\n" - "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x\n", + "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x, " + "mv=y:%d,x:%d\n", signabs(t->warpmv.matrix[0]), signabs(t->warpmv.matrix[1]), signabs(t->warpmv.matrix[2]), signabs(t->warpmv.matrix[3]), signabs(t->warpmv.matrix[4]), signabs(t->warpmv.matrix[5]), signabs(t->warpmv.alpha), signabs(t->warpmv.beta), signabs(t->warpmv.gamma), - signabs(t->warpmv.delta)); + signabs(t->warpmv.delta), + b->mv[0].y, b->mv[0].x); #undef signabs + if (f->frame_thread.pass) { + if (t->warpmv.type == DAV1D_WM_TYPE_AFFINE) { + b->matrix[0] = t->warpmv.matrix[2] - 0x10000; + b->matrix[1] = t->warpmv.matrix[3]; + b->matrix[2] = t->warpmv.matrix[4]; + b->matrix[3] = t->warpmv.matrix[5] - 0x10000; + } else { + b->matrix[0] = SHRT_MIN; + } + } } if (DEBUG_BLOCK_INFO) printf("Post-motionmode[%d]: r=%d [mask: 0x%" PRIu64 "x/0x%" PRIu64 "x]\n", b->motion_mode, ts->msac.rng, mask[0], @@ -1876,28 +1920,29 @@ { const int is_globalmv = b->inter_mode == (is_comp ? GLOBALMV_GLOBALMV : GLOBALMV); const uint8_t (*const lf_lvls)[8][2] = (const uint8_t (*)[8][2]) &ts->lflvl[b->seg_id][0][b->ref[0] + 1][!is_globalmv]; - dav1d_create_lf_mask_inter(t->lf_mask, f->lf.level, f->b4_stride, - lf_lvls, t->bx, t->by, f->w4, f->h4, - b->skip, bs, b->tx_split, b->uvtx, - f->cur.p.layout, + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; + dav1d_create_lf_mask_inter(t->lf_mask, f->lf.level, f->b4_stride, lf_lvls, + t->bx, t->by, f->w4, f->h4, b->skip, bs, + f->frame_hdr->segmentation.lossless[b->seg_id] ? + (enum RectTxfmSize) TX_4X4 : b->max_ytx, + tx_split, b->uvtx, f->cur.p.layout, &t->a->tx_lpf_y[bx4], &t->l.tx_lpf_y[by4], has_chroma ? &t->a->tx_lpf_uv[cbx4] : NULL, has_chroma ? &t->l.tx_lpf_uv[cby4] : NULL); } // context updates if (is_comp) { - splat_tworef_mv(f->mvs, f->b4_stride, t->by, t->bx, bs, - b->inter_mode, b->ref[0], b->ref[1], - b->mv[0], b->mv[1]); + splat_tworef_mv(&t->rt, t->by, t->bx, bs, b->inter_mode, + (refmvs_refpair) { .ref = { b->ref[0], b->ref[1] }}, + (refmvs_mvpair) { .mv = { [0] = b->mv[0], [1] = b->mv[1] }}); } else { - splat_oneref_mv(f->mvs, f->b4_stride, t->by, t->bx, bs, - b->inter_mode, b->ref[0], b->mv[0], - b->interintra_type); + splat_oneref_mv(&t->rt, t->by, t->bx, bs, b->inter_mode, + b->ref[0], b->mv[0], b->interintra_type); } #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ rep_macro(type, t->dir skip_mode, off, mul * b->skip_mode); \ @@ -2348,11 +2393,11 @@ ts->tiling.row_start = row_sb_start << sb_shift; ts->tiling.row_end = imin(row_sb_end << sb_shift, f->bh); // Reference Restoration Unit (used for exp coding) int sb_idx, unit_idx; - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { // vertical components only sb_idx = (ts->tiling.row_start >> 5) * f->sr_sb128w; unit_idx = (ts->tiling.row_start & 16) >> 3; } else { sb_idx = (ts->tiling.row_start >> 5) * f->sb128w + col_sb128_start; @@ -2361,11 +2406,11 @@ } for (int p = 0; p < 3; p++) { if (!((f->lf.restore_planes >> p) & 1U)) continue; - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { const int ss_hor = p && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; const int d = f->frame_hdr->super_res.width_scale_denominator; const int unit_size_log2 = f->frame_hdr->restoration.unit_size[!!p]; const int rnd = (8 << unit_size_log2) - 1, shift = unit_size_log2 + 3; const int x = ((4 * ts->tiling.col_start * d >> ss_hor) + rnd) >> shift; @@ -2469,10 +2514,17 @@ const int sb_step = f->sb_step; const int tile_row = ts->tiling.row, tile_col = ts->tiling.col; const int col_sb_start = f->frame_hdr->tiling.col_start_sb[tile_col]; const int col_sb128_start = col_sb_start >> !f->seq_hdr->sb128; + if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { + dav1d_refmvs_tile_sbrow_init(&t->rt, &f->rf, ts->tiling.col_start, + ts->tiling.col_end, ts->tiling.row_start, + ts->tiling.row_end, t->by >> f->sb_shift, + ts->tiling.row); + } + reset_context(&t->l, !(f->frame_hdr->frame_type & 1), f->frame_thread.pass); if (f->frame_thread.pass == 2) { for (t->bx = ts->tiling.col_start, t->a = f->a + col_sb128_start + tile_row * f->sb128w; t->bx < ts->tiling.col_end; t->bx += sb_step) @@ -2489,20 +2541,20 @@ } // error out on symbol decoder overread if (ts->msac.cnt < -15) return 1; - if (c->n_fc > 1 && f->frame_hdr->use_ref_frame_mvs) { - for (int n = 0; n < 7; n++) + if (f->n_tc > 1 && f->frame_hdr->use_ref_frame_mvs) { + if (c->n_fc > 1) for (int n = 0; n < 7; n++) if (dav1d_thread_picture_wait(&f->refp[n], 4 * (t->by + sb_step), PLANE_TYPE_BLOCK)) { return 1; } - dav1d_init_ref_mv_tile_row(f->libaom_cm, - ts->tiling.col_start, ts->tiling.col_end, - t->by, imin(t->by + sb_step, f->bh)); + dav1d_refmvs_load_tmvs(&f->rf, ts->tiling.row, + ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, + t->by >> 1, (t->by + sb_step) >> 1); } memset(t->pal_sz_uv[1], 0, sizeof(*t->pal_sz_uv)); const int sb128y = t->by >> 5; for (t->bx = ts->tiling.col_start, t->a = f->a + col_sb128_start + tile_row * f->sb128w, t->lf_mask = f->lf.mask + sb128y * f->sb128w + col_sb128_start; @@ -2541,11 +2593,11 @@ // restoration unit if (y && y + half_unit > h) continue; const enum Dav1dRestorationType frame_type = f->frame_hdr->restoration.type[p]; - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { const int w = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; const int n_units = imax(1, (w + half_unit) >> unit_size_log2); const int d = f->frame_hdr->super_res.width_scale_denominator; const int rnd = unit_size * 8 - 1, shift = unit_size_log2 + 3; @@ -2580,10 +2632,16 @@ t->a++; t->lf_mask++; } } + if (f->n_tc > 1 && f->frame_hdr->frame_type & 1) { + dav1d_refmvs_save_tmvs(&t->rt, + ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, + t->by >> 1, (t->by + sb_step) >> 1); + } + // backup pre-loopfilter pixels for intra prediction of the next sbrow if (f->frame_thread.pass != 1) f->bd_fn.backup_ipred_edge(t); // backup t->a/l.tx_lpf_y/uv at tile boundaries to use them to "fix" @@ -2761,28 +2819,46 @@ f->frame_thread.pal_sz = f->frame_thread.pal_idx_sz = 0; } } // update allocation of block contexts for above - const int line_sz = (int)f->b4_stride << hbd; - if (line_sz != f->lf.line_sz) { - dav1d_freep_aligned(&f->lf.cdef_line[0][0][0]); - uint8_t *ptr = dav1d_alloc_aligned(line_sz * 4 * 12, 32); + const ptrdiff_t y_stride = f->cur.stride[0], uv_stride = f->cur.stride[1]; + if (y_stride != f->lf.cdef_line_sz[0] || uv_stride != f->lf.cdef_line_sz[1]) { + dav1d_free_aligned(f->lf.cdef_line_buf); + size_t alloc_sz = 64; + alloc_sz += (y_stride < 0 ? -y_stride : y_stride ) * 4; + alloc_sz += (uv_stride < 0 ? -uv_stride : uv_stride) * 8; + uint8_t *ptr = f->lf.cdef_line_buf = dav1d_alloc_aligned(alloc_sz, 32); if (!ptr) { - f->lf.line_sz = 0; + f->lf.cdef_line_sz[0] = f->lf.cdef_line_sz[1] = 0; goto error; } - for (int pl = 0; pl <= 2; pl++) { - f->lf.cdef_line[0][pl][0] = ptr + line_sz * 4 * 0; - f->lf.cdef_line[0][pl][1] = ptr + line_sz * 4 * 1; - f->lf.cdef_line[1][pl][0] = ptr + line_sz * 4 * 2; - f->lf.cdef_line[1][pl][1] = ptr + line_sz * 4 * 3; - ptr += line_sz * 4 * 4; + ptr += 32; + if (y_stride < 0) { + f->lf.cdef_line[0][0] = ptr - y_stride * 1; + f->lf.cdef_line[1][0] = ptr - y_stride * 3; + ptr -= y_stride * 4; + } else { + f->lf.cdef_line[0][0] = ptr + y_stride * 0; + f->lf.cdef_line[1][0] = ptr + y_stride * 2; + ptr += y_stride * 4; + } + if (uv_stride < 0) { + f->lf.cdef_line[0][1] = ptr - uv_stride * 1; + f->lf.cdef_line[0][2] = ptr - uv_stride * 3; + f->lf.cdef_line[1][1] = ptr - uv_stride * 5; + f->lf.cdef_line[1][2] = ptr - uv_stride * 7; + } else { + f->lf.cdef_line[0][1] = ptr + uv_stride * 0; + f->lf.cdef_line[0][2] = ptr + uv_stride * 2; + f->lf.cdef_line[1][1] = ptr + uv_stride * 4; + f->lf.cdef_line[1][2] = ptr + uv_stride * 6; } - f->lf.line_sz = line_sz; + f->lf.cdef_line_sz[0] = (int) y_stride; + f->lf.cdef_line_sz[1] = (int) uv_stride; } const int lr_line_sz = ((f->sr_cur.p.p.w + 31) & ~31) << hbd; if (lr_line_sz != f->lf.lr_line_sz) { dav1d_freep_aligned(&f->lf.lr_lpf_line[0]); @@ -2875,24 +2951,14 @@ f->lf.re_sz = re_sz; } // init ref mvs if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { - f->mvs = f->mvs_ref->data; - const int order_hint_n_bits = f->seq_hdr->order_hint * f->seq_hdr->order_hint_n_bits; - const int ret = dav1d_init_ref_mv_common(f->libaom_cm, f->bw >> 1, f->bh >> 1, - f->b4_stride, f->seq_hdr->sb128, - f->mvs, f->ref_mvs, - f->cur.frame_hdr->frame_offset, - f->refpoc, - f->refrefpoc, f->frame_hdr->gmv, - f->frame_hdr->hp, f->frame_hdr->force_integer_mv, - f->frame_hdr->use_ref_frame_mvs, - order_hint_n_bits); + const int ret = + dav1d_refmvs_init_frame(&f->rf, f->seq_hdr, f->frame_hdr, + f->refpoc, f->mvs, f->refrefpoc, f->ref_mvs, f->n_tc); if (ret < 0) goto error; - if (c->n_fc == 1 && f->frame_hdr->use_ref_frame_mvs) - dav1d_init_ref_mv_tile_row(f->libaom_cm, 0, f->bw, 0, f->bh); } retval = DAV1D_ERR(EINVAL); // setup dequant tables init_quant_tables(f->seq_hdr, f->frame_hdr, f->frame_hdr->quant.yac, f->dq); @@ -2942,18 +3008,23 @@ f->jnt_weights[i][j] = quant_dist_lookup_table[k][order]; } } } - // init loopfilter pointers + /* Init loopfilter pointers. Increasing NULL pointers is technically UB, + * so just point the chroma pointers in 4:0:0 to the luma plane here to + * avoid having additional in-loop branches in various places. We never + * dereference those pointers so it doesn't really matter what they + * point at, as long as the pointers are valid. */ + const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400; f->lf.mask_ptr = f->lf.mask; f->lf.p[0] = f->cur.data[0]; - f->lf.p[1] = f->cur.data[1]; - f->lf.p[2] = f->cur.data[2]; + f->lf.p[1] = f->cur.data[has_chroma ? 1 : 0]; + f->lf.p[2] = f->cur.data[has_chroma ? 2 : 0]; f->lf.sr_p[0] = f->sr_cur.p.data[0]; - f->lf.sr_p[1] = f->sr_cur.p.data[1]; - f->lf.sr_p[2] = f->sr_cur.p.data[2]; + f->lf.sr_p[1] = f->sr_cur.p.data[has_chroma ? 1 : 0]; + f->lf.sr_p[2] = f->sr_cur.p.data[has_chroma ? 2 : 0]; f->lf.tile_row = 1; dav1d_cdf_thread_wait(&f->in_cdf); if (f->frame_hdr->refresh_context) dav1d_cdf_thread_copy(f->out_cdf.data.cdf, &f->in_cdf); @@ -3024,15 +3095,30 @@ imin(f->frame_hdr->tiling.row_start_sb[tile_row + 1], f->sbh); for (int sby = f->frame_hdr->tiling.row_start_sb[tile_row]; sby < sbh_end; sby++) { t->by = sby << (4 + f->seq_hdr->sb128); + const int by_end = (t->by + f->sb_step) >> 1; + if (f->frame_thread.pass <= 1 && f->frame_hdr->use_ref_frame_mvs) { + if (c->n_fc > 1) for (int n = 0; n < 7; n++) + if (dav1d_thread_picture_wait(&f->refp[n], + 4 * (t->by + f->sb_step), + PLANE_TYPE_BLOCK)) + { + return 1; + } + dav1d_refmvs_load_tmvs(&f->rf, tile_row, + 0, f->bw >> 1, t->by >> 1, by_end); + } for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { t->ts = &f->ts[tile_row * f->frame_hdr->tiling.cols + tile_col]; if (dav1d_decode_tile_sbrow(t)) goto error; } + if (f->frame_thread.pass <= 1 && f->frame_hdr->frame_type & 1) { + dav1d_refmvs_save_tmvs(&t->rt, 0, f->bw >> 1, t->by >> 1, by_end); + } // loopfilter + cdef + restoration if (f->frame_thread.pass != 1) f->bd_fn.filter_sbrow(f, sby); dav1d_thread_picture_signal(&f->sr_cur, (sby + 1) * f->sb_step * 4, @@ -3218,11 +3304,11 @@ #define assign_bitdepth_case(bd) \ dav1d_cdef_dsp_init_##bd##bpc(&dsp->cdef); \ dav1d_intra_pred_dsp_init_##bd##bpc(&dsp->ipred); \ dav1d_itx_dsp_init_##bd##bpc(&dsp->itx); \ dav1d_loop_filter_dsp_init_##bd##bpc(&dsp->lf); \ - dav1d_loop_restoration_dsp_init_##bd##bpc(&dsp->lr); \ + dav1d_loop_restoration_dsp_init_##bd##bpc(&dsp->lr, bpc); \ dav1d_mc_dsp_init_##bd##bpc(&dsp->mc); \ dav1d_film_grain_dsp_init_##bd##bpc(&dsp->fg); \ break #if CONFIG_8BPC case 8: @@ -3299,11 +3385,12 @@ } else { f->svc[i][0].scale = 0; } f->gmv_warp_allowed[i] = f->frame_hdr->gmv[i].type > DAV1D_WM_TYPE_TRANSLATION && !f->frame_hdr->force_integer_mv && - !dav1d_get_shear_params(&f->frame_hdr->gmv[i]); + !dav1d_get_shear_params(&f->frame_hdr->gmv[i]) && + !f->svc[i][0].scale; } } // setup entropy if (f->frame_hdr->primary_ref_frame == DAV1D_PRIMARY_REF_NONE) { @@ -3336,18 +3423,18 @@ // allocate frame res = dav1d_thread_picture_alloc(c, f, bpc); if (res < 0) goto error; - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { res = dav1d_picture_alloc_copy(c, &f->cur, f->frame_hdr->width[0], &f->sr_cur.p); if (res < 0) goto error; } else { dav1d_picture_ref(&f->cur, &f->sr_cur.p); } - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { f->resize_step[0] = scale_fac(f->cur.p.w, f->sr_cur.p.p.w); const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; const int in_cw = (f->cur.p.w + ss_hor) >> ss_hor; const int out_cw = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; f->resize_step[1] = scale_fac(in_cw, out_cw); @@ -3376,11 +3463,11 @@ f->b4_stride = (f->bw + 31) & ~31; f->bitdepth_max = (1 << f->cur.p.bpc) - 1; // ref_mvs if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { - f->mvs_ref = dav1d_ref_create(f->sb128h * 32 * f->b4_stride * + f->mvs_ref = dav1d_ref_create(f->sb128h * 16 * (f->b4_stride >> 1) * sizeof(*f->mvs)); if (!f->mvs_ref) { res = DAV1D_ERR(ENOMEM); goto error; } diff --git a/third_party/dav1d/src/env.h b/third_party/dav1d/src/env.h --- a/third_party/dav1d/src/env.h +++ b/third_party/dav1d/src/env.h @@ -31,11 +31,11 @@ #include #include #include #include "src/levels.h" -#include "src/ref_mvs.h" +#include "src/refmvs.h" #include "src/tables.h" typedef struct BlockContext { uint8_t ALIGN(mode[32], 8); uint8_t ALIGN(lcoef[32], 8); @@ -426,11 +426,11 @@ cnt[1] += cnt[2]; return cnt[0] == cnt[1] ? 1 : cnt[0] < cnt[1] ? 0 : 2; } -static inline int get_drl_context(const candidate_mv *const ref_mv_stack, +static inline int get_drl_context(const refmvs_candidate *const ref_mv_stack, const int ref_idx) { if (ref_mv_stack[ref_idx].weight >= 640) return ref_mv_stack[ref_idx + 1].weight < 640; @@ -458,10 +458,26 @@ *seg_ctx = 0; return have_left ? cur_seg_map[-1] : have_top ? cur_seg_map[-stride] : 0; } } +static inline void fix_int_mv_precision(mv *const mv) { + mv->x = (mv->x - (mv->x >> 15) + 3) & ~7U; + mv->y = (mv->y - (mv->y >> 15) + 3) & ~7U; +} + +static inline void fix_mv_precision(const Dav1dFrameHeader *const hdr, + mv *const mv) +{ + if (hdr->force_integer_mv) { + fix_int_mv_precision(mv); + } else if (!hdr->hp) { + mv->x = (mv->x - (mv->x >> 15)) & ~1U; + mv->y = (mv->y - (mv->y >> 15)) & ~1U; + } +} + static inline mv get_gmv_2d(const Dav1dWarpedMotionParams *const gmv, const int bx4, const int by4, const int bw4, const int bh4, const Dav1dFrameHeader *const hdr) { @@ -478,20 +494,27 @@ gmv->matrix[3] * y + gmv->matrix[0]; const int yc = (gmv->matrix[5] - (1 << 16)) * y + gmv->matrix[4] * x + gmv->matrix[1]; const int shift = 16 - (3 - !hdr->hp); const int round = (1 << shift) >> 1; - return (mv) { + mv res = (mv) { .y = apply_sign(((abs(yc) + round) >> shift) << !hdr->hp, yc), .x = apply_sign(((abs(xc) + round) >> shift) << !hdr->hp, xc), }; + if (hdr->force_integer_mv) + fix_int_mv_precision(&res); + return res; } - case DAV1D_WM_TYPE_TRANSLATION: - return (mv) { + case DAV1D_WM_TYPE_TRANSLATION: { + mv res = (mv) { .y = gmv->matrix[0] >> 13, .x = gmv->matrix[1] >> 13, }; + if (hdr->force_integer_mv) + fix_int_mv_precision(&res); + return res; + } case DAV1D_WM_TYPE_IDENTITY: return (mv) { .x = 0, .y = 0 }; } } diff --git a/third_party/dav1d/src/ext/x86/x86inc.asm b/third_party/dav1d/src/ext/x86/x86inc.asm --- a/third_party/dav1d/src/ext/x86/x86inc.asm +++ b/third_party/dav1d/src/ext/x86/x86inc.asm @@ -649,12 +649,14 @@ %endif ;====================================================================== %if WIN64 == 0 %macro WIN64_SPILL_XMM 1 + %assign xmm_regs_used %1 %endmacro %macro WIN64_RESTORE_XMM 0 + %assign xmm_regs_used 0 %endmacro %macro WIN64_PUSH_XMM 0 %endmacro %endif @@ -822,37 +824,38 @@ %endif %endmacro ; cpuflags -%assign cpuflags_mmx (1<<0) -%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx -%assign cpuflags_3dnow (1<<2) | cpuflags_mmx -%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow -%assign cpuflags_sse (1<<4) | cpuflags_mmx2 -%assign cpuflags_sse2 (1<<5) | cpuflags_sse -%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 -%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2 -%assign cpuflags_sse3 (1<<8) | cpuflags_sse2 -%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3 -%assign cpuflags_sse4 (1<<10)| cpuflags_ssse3 -%assign cpuflags_sse42 (1<<11)| cpuflags_sse4 -%assign cpuflags_aesni (1<<12)| cpuflags_sse42 -%assign cpuflags_gfni (1<<13)| cpuflags_sse42 -%assign cpuflags_avx (1<<14)| cpuflags_sse42 -%assign cpuflags_xop (1<<15)| cpuflags_avx -%assign cpuflags_fma4 (1<<16)| cpuflags_avx -%assign cpuflags_fma3 (1<<17)| cpuflags_avx -%assign cpuflags_bmi1 (1<<18)| cpuflags_avx|cpuflags_lzcnt -%assign cpuflags_bmi2 (1<<19)| cpuflags_bmi1 -%assign cpuflags_avx2 (1<<20)| cpuflags_fma3|cpuflags_bmi2 -%assign cpuflags_avx512 (1<<21)| cpuflags_avx2 ; F, CD, BW, DQ, VL +%assign cpuflags_mmx (1<<0) +%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx +%assign cpuflags_3dnow (1<<2) | cpuflags_mmx +%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow +%assign cpuflags_sse (1<<4) | cpuflags_mmx2 +%assign cpuflags_sse2 (1<<5) | cpuflags_sse +%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 +%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2 +%assign cpuflags_sse3 (1<<8) | cpuflags_sse2 +%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3 +%assign cpuflags_sse4 (1<<10) | cpuflags_ssse3 +%assign cpuflags_sse42 (1<<11) | cpuflags_sse4 +%assign cpuflags_aesni (1<<12) | cpuflags_sse42 +%assign cpuflags_gfni (1<<13) | cpuflags_sse42 +%assign cpuflags_avx (1<<14) | cpuflags_sse42 +%assign cpuflags_xop (1<<15) | cpuflags_avx +%assign cpuflags_fma4 (1<<16) | cpuflags_avx +%assign cpuflags_fma3 (1<<17) | cpuflags_avx +%assign cpuflags_bmi1 (1<<18) | cpuflags_avx|cpuflags_lzcnt +%assign cpuflags_bmi2 (1<<19) | cpuflags_bmi1 +%assign cpuflags_avx2 (1<<20) | cpuflags_fma3|cpuflags_bmi2 +%assign cpuflags_avx512 (1<<21) | cpuflags_avx2 ; F, CD, BW, DQ, VL +%assign cpuflags_avx512icl (1<<22) | cpuflags_avx512|cpuflags_gfni ; VNNI, IFMA, VBMI, VBMI2, VPOPCNTDQ, BITALG, VAES, VPCLMULQDQ -%assign cpuflags_cache32 (1<<22) -%assign cpuflags_cache64 (1<<23) -%assign cpuflags_aligned (1<<24) ; not a cpu feature, but a function variant -%assign cpuflags_atom (1<<25) +%assign cpuflags_cache32 (1<<23) +%assign cpuflags_cache64 (1<<24) +%assign cpuflags_aligned (1<<25) ; not a cpu feature, but a function variant +%assign cpuflags_atom (1<<26) ; Returns a boolean value expressing whether or not the specified cpuflag is enabled. %define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1) %define notcpuflag(x) (cpuflag(x) ^ 1) diff --git a/third_party/dav1d/src/fg_apply_tmpl.c b/third_party/dav1d/src/fg_apply_tmpl.c --- a/third_party/dav1d/src/fg_apply_tmpl.c +++ b/third_party/dav1d/src/fg_apply_tmpl.c @@ -120,21 +120,36 @@ // Copy over the non-modified planes // TODO: eliminate in favor of per-plane refs assert(out->stride[0] == in->stride[0]); if (!data->num_y_points) { - memcpy(out->data[0], in->data[0], out->p.h * out->stride[0]); + const ptrdiff_t stride = out->stride[0]; + const ptrdiff_t sz = out->p.h * stride; + if (sz < 0) + memcpy((uint8_t*) out->data[0] + sz - stride, + (uint8_t*) in->data[0] + sz - stride, -sz); + else + memcpy(out->data[0], in->data[0], sz); } - if (in->p.layout != DAV1D_PIXEL_LAYOUT_I400) { + if (in->p.layout != DAV1D_PIXEL_LAYOUT_I400 && !data->chroma_scaling_from_luma) { assert(out->stride[1] == in->stride[1]); - for (int i = 0; i < 2; i++) { - if (!data->num_uv_points[i] && !data->chroma_scaling_from_luma) { - const int suby = in->p.layout == DAV1D_PIXEL_LAYOUT_I420; - memcpy(out->data[1+i], in->data[1+i], - (out->p.h >> suby) * out->stride[1]); - } + const int ss_ver = in->p.layout == DAV1D_PIXEL_LAYOUT_I420; + const ptrdiff_t stride = out->stride[1]; + const ptrdiff_t sz = (out->p.h * stride) >> ss_ver; + if (sz < 0) { + if (!data->num_uv_points[0]) + memcpy((uint8_t*) out->data[1] + sz - stride, + (uint8_t*) in->data[1] + sz - stride, -sz); + if (!data->num_uv_points[1]) + memcpy((uint8_t*) out->data[2] + sz - stride, + (uint8_t*) in->data[2] + sz - stride, -sz); + } else { + if (!data->num_uv_points[0]) + memcpy(out->data[1], in->data[1], sz); + if (!data->num_uv_points[1]) + memcpy(out->data[2], in->data[2], sz); } } // Synthesize grain for the affected planes const int rows = (out->p.h + 31) >> 5; diff --git a/third_party/dav1d/src/film_grain_tmpl.c b/third_party/dav1d/src/film_grain_tmpl.c --- a/third_party/dav1d/src/film_grain_tmpl.c +++ b/third_party/dav1d/src/film_grain_tmpl.c @@ -41,11 +41,11 @@ *state = (r >> 1) | (bit << 15); return (*state >> (16 - bits)) & ((1 << bits) - 1); } -static inline int round2(const int x, const int shift) { +static inline int round2(const int x, const uint64_t shift) { return (x + ((1 << shift) >> 1)) >> shift; } static void generate_grain_y_c(entry buf[][GRAIN_WIDTH], const Dav1dFilmGrainData *const data diff --git a/third_party/dav1d/src/internal.h b/third_party/dav1d/src/internal.h --- a/third_party/dav1d/src/internal.h +++ b/third_party/dav1d/src/internal.h @@ -52,11 +52,11 @@ #include "src/looprestoration.h" #include "src/mc.h" #include "src/msac.h" #include "src/picture.h" #include "src/recon.h" -#include "src/ref_mvs.h" +#include "src/refmvs.h" #include "src/thread.h" typedef struct Dav1dDSPContext { Dav1dFilmGrainDSPContext fg; Dav1dIntraPredDSPContext ipred; @@ -144,11 +144,11 @@ Dav1dFrameHeader *frame_hdr; Dav1dThreadPicture refp[7]; Dav1dPicture cur; // during block coding / reconstruction Dav1dThreadPicture sr_cur; // after super-resolution upscaling Dav1dRef *mvs_ref; - refmvs *mvs, *ref_mvs[7]; + refmvs_temporal_block *mvs, *ref_mvs[7]; Dav1dRef *ref_mvs_ref[7]; Dav1dRef *cur_segmap_ref, *prev_segmap_ref; uint8_t *cur_segmap; const uint8_t *prev_segmap; unsigned refpoc[7], refrefpoc[7][7]; @@ -185,11 +185,11 @@ int w4, h4, bw, bh, sb128w, sb128h, sbh, sb_shift, sb_step, sr_sb128w; uint16_t dq[DAV1D_MAX_SEGMENTS][3 /* plane */][2 /* dc/ac */]; const uint8_t *qm[2 /* is_1d */][N_RECT_TX_SIZES][3 /* plane */]; BlockContext *a; int a_sz /* w*tile_rows */; - AV1_COMMON *libaom_cm; // FIXME + refmvs_frame rf; uint8_t jnt_weights[7][7]; int bitdepth_max; struct { struct thread_data td; @@ -214,16 +214,18 @@ struct { uint8_t (*level)[4]; Av1Filter *mask; Av1Restoration *lr_mask; int top_pre_cdef_toggle; - int mask_sz /* w*h */, lr_mask_sz, line_sz /* w */, lr_line_sz, re_sz /* h */; + int mask_sz /* w*h */, lr_mask_sz, cdef_line_sz[2] /* stride */; + int lr_line_sz, re_sz /* h */; ALIGN(Av1FilterLUT lim_lut, 16); int last_sharpness; uint8_t lvl[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */]; uint8_t *tx_lpf_right_edge[2]; - pixel *cdef_line[2 /* pre, post */][3 /* plane */][2 /* y */]; + uint8_t *cdef_line_buf; + pixel *cdef_line[2 /* pre, post */][3 /* plane */]; pixel *lr_lpf_line[3 /* plane */]; // in-loop filter per-frame state keeping int tile_row; // for carry-over at tile row edges pixel *p[3], *sr_p[3]; @@ -286,11 +288,12 @@ // FIXME types can be changed to pixel (and dynamically allocated) // which would make copy/assign operations slightly faster? uint16_t al_pal[2 /* a/l */][32 /* bx/y4 */][3 /* plane */][8 /* palette_idx */]; uint8_t pal_sz_uv[2 /* a/l */][32 /* bx4/by4 */]; uint8_t txtp_map[32 * 32]; // inter-only - ALIGN(union, 32) { + refmvs_tile rt; + ALIGN(union, 64) { struct { union { uint8_t lap_8bpc [128 * 32]; uint16_t lap_16bpc[128 * 32]; struct { diff --git a/third_party/dav1d/src/ipred_prepare.h b/third_party/dav1d/src/ipred_prepare.h --- a/third_party/dav1d/src/ipred_prepare.h +++ b/third_party/dav1d/src/ipred_prepare.h @@ -64,11 +64,11 @@ * topleft_out is a pointer to scratch memory that will be filled with the edge * pixels. The memory array should have space to be indexed in the [-2*w,2*w] * range, in the following order: * - [0] will be the top/left edge pixel; * - [1..w] will be the top edge pixels (1 being left-most, w being right-most); - * - [w+1..w*w] will be the top/right edge pixels; + * - [w+1..2*w] will be the top/right edge pixels; * - [-1..-w] will be the left edge pixels (-1 being top-most, -w being bottom- * most); * - [-w-1..-2*w] will be the bottom/left edge pixels. * Each edge may remain uninitialized if it is not used by the returned mode * index. If edges are not available (because the edge position is outside the diff --git a/third_party/dav1d/src/itx_1d.h b/third_party/dav1d/src/itx_1d.h new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/itx_1d.h @@ -0,0 +1,59 @@ +/* + * Copyright © 2018-2019, VideoLAN and dav1d authors + * Copyright © 2018-2019, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#ifndef DAV1D_SRC_ITX_1D_H +#define DAV1D_SRC_ITX_1D_H + +#define decl_itx_1d_fn(name) \ +void (name)(int32_t *c, ptrdiff_t stride, int min, int max) +typedef decl_itx_1d_fn(*itx_1d_fn); + +decl_itx_1d_fn(dav1d_inv_dct4_1d_c); +decl_itx_1d_fn(dav1d_inv_dct8_1d_c); +decl_itx_1d_fn(dav1d_inv_dct16_1d_c); +decl_itx_1d_fn(dav1d_inv_dct32_1d_c); +decl_itx_1d_fn(dav1d_inv_dct64_1d_c); + +decl_itx_1d_fn(dav1d_inv_adst4_1d_c); +decl_itx_1d_fn(dav1d_inv_adst8_1d_c); +decl_itx_1d_fn(dav1d_inv_adst16_1d_c); + +decl_itx_1d_fn(dav1d_inv_flipadst4_1d_c); +decl_itx_1d_fn(dav1d_inv_flipadst8_1d_c); +decl_itx_1d_fn(dav1d_inv_flipadst16_1d_c); + +decl_itx_1d_fn(dav1d_inv_identity4_1d_c); +decl_itx_1d_fn(dav1d_inv_identity8_1d_c); +decl_itx_1d_fn(dav1d_inv_identity16_1d_c); +decl_itx_1d_fn(dav1d_inv_identity32_1d_c); + +void dav1d_inv_wht4_1d_c(int32_t *c, ptrdiff_t stride); + +#endif /* DAV1D_SRC_ITX_1D_H */ diff --git a/third_party/dav1d/src/itx_1d.c b/third_party/dav1d/src/itx_1d.c --- a/third_party/dav1d/src/itx_1d.c +++ b/third_party/dav1d/src/itx_1d.c @@ -1,8 +1,8 @@ /* - * Copyright © 2018, VideoLAN and dav1d authors - * Copyright © 2018, Two Orioles, LLC + * Copyright © 2018-2019, VideoLAN and dav1d authors + * Copyright © 2018-2019, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * @@ -28,11 +28,13 @@ #include "config.h" #include #include -#include "common/attributes.h" +#include "common/intops.h" + +#include "src/itx_1d.h" #define CLIP(a) iclip(a, min, max) /* * In some places, we use the pattern like this: @@ -58,86 +60,129 @@ * integer overflows, since these are not considered valid AV1 anyway, and in * e.g. x86 assembly, integer overflows are not considered UB, but they merely * wrap around. */ -static void NOINLINE -inv_dct4_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int max) +static NOINLINE void +inv_dct4_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, const int tx64) { - const int min = -max - 1; - const int in0 = in[0 * in_s], in1 = in[1 * in_s]; - const int in2 = in[2 * in_s], in3 = in[3 * in_s]; + assert(stride > 0); + const int in0 = c[0 * stride], in1 = c[1 * stride]; + + int t0, t1, t2, t3; + if (tx64) { + t0 = t1 = (in0 * 181 + 128) >> 8; + t2 = (in1 * 1567 + 2048) >> 12; + t3 = (in1 * 3784 + 2048) >> 12; + } else { + const int in2 = c[2 * stride], in3 = c[3 * stride]; - int t0 = ((in0 + in2) * 181 + 128) >> 8; - int t1 = ((in0 - in2) * 181 + 128) >> 8; - int t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; - int t3 = ((in1 * (3784 - 4096) + in3 * 1567 + 2048) >> 12) + in1; + t0 = ((in0 + in2) * 181 + 128) >> 8; + t1 = ((in0 - in2) * 181 + 128) >> 8; + t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; + t3 = ((in1 * (3784 - 4096) + in3 * 1567 + 2048) >> 12) + in1; + } - out[0 * out_s] = CLIP(t0 + t3); - out[1 * out_s] = CLIP(t1 + t2); - out[2 * out_s] = CLIP(t1 - t2); - out[3 * out_s] = CLIP(t0 - t3); + c[0 * stride] = CLIP(t0 + t3); + c[1 * stride] = CLIP(t1 + t2); + c[2 * stride] = CLIP(t1 - t2); + c[3 * stride] = CLIP(t0 - t3); +} + +void dav1d_inv_dct4_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + inv_dct4_1d_internal_c(c, stride, min, max, 0); } -static void NOINLINE -inv_dct8_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int max) +static NOINLINE void +inv_dct8_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, const int tx64) { - const int min = -max - 1; - coef tmp[4]; + assert(stride > 0); + inv_dct4_1d_internal_c(c, stride << 1, min, max, tx64); + + const int in1 = c[1 * stride], in3 = c[3 * stride]; - inv_dct4_1d(in, in_s * 2, tmp, 1, max); - - const int in1 = in[1 * in_s], in3 = in[3 * in_s]; - const int in5 = in[5 * in_s], in7 = in[7 * in_s]; - - int t4a = ((in1 * 799 - in7 * (4017 - 4096) + 2048) >> 12) - in7; - int t5a = (in5 * 1703 - in3 * 1138 + 1024) >> 11; - int t6a = (in5 * 1138 + in3 * 1703 + 1024) >> 11; - int t7a = ((in1 * (4017 - 4096) + in7 * 799 + 2048) >> 12) + in1; + int t4a, t5a, t6a, t7a; + if (tx64) { + t4a = (in1 * 799 + 2048) >> 12; + t5a = (in3 * -2276 + 2048) >> 12; + t6a = (in3 * 3406 + 2048) >> 12; + t7a = (in1 * 4017 + 2048) >> 12; + } else { + const int in5 = c[5 * stride], in7 = c[7 * stride]; - int t4 = CLIP(t4a + t5a); - t5a = CLIP(t4a - t5a); - int t7 = CLIP(t7a + t6a); - t6a = CLIP(t7a - t6a); + t4a = ((in1 * 799 - in7 * (4017 - 4096) + 2048) >> 12) - in7; + t5a = (in5 * 1703 - in3 * 1138 + 1024) >> 11; + t6a = (in5 * 1138 + in3 * 1703 + 1024) >> 11; + t7a = ((in1 * (4017 - 4096) + in7 * 799 + 2048) >> 12) + in1; + } - int t5 = ((t6a - t5a) * 181 + 128) >> 8; - int t6 = ((t6a + t5a) * 181 + 128) >> 8; + const int t4 = CLIP(t4a + t5a); + t5a = CLIP(t4a - t5a); + const int t7 = CLIP(t7a + t6a); + t6a = CLIP(t7a - t6a); + + const int t5 = ((t6a - t5a) * 181 + 128) >> 8; + const int t6 = ((t6a + t5a) * 181 + 128) >> 8; - out[0 * out_s] = CLIP(tmp[0] + t7); - out[1 * out_s] = CLIP(tmp[1] + t6); - out[2 * out_s] = CLIP(tmp[2] + t5); - out[3 * out_s] = CLIP(tmp[3] + t4); - out[4 * out_s] = CLIP(tmp[3] - t4); - out[5 * out_s] = CLIP(tmp[2] - t5); - out[6 * out_s] = CLIP(tmp[1] - t6); - out[7 * out_s] = CLIP(tmp[0] - t7); + const int t0 = c[0 * stride]; + const int t1 = c[2 * stride]; + const int t2 = c[4 * stride]; + const int t3 = c[6 * stride]; + + c[0 * stride] = CLIP(t0 + t7); + c[1 * stride] = CLIP(t1 + t6); + c[2 * stride] = CLIP(t2 + t5); + c[3 * stride] = CLIP(t3 + t4); + c[4 * stride] = CLIP(t3 - t4); + c[5 * stride] = CLIP(t2 - t5); + c[6 * stride] = CLIP(t1 - t6); + c[7 * stride] = CLIP(t0 - t7); } -static void NOINLINE -inv_dct16_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int max) +void dav1d_inv_dct8_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) { - const int min = -max - 1; - coef tmp[8]; + inv_dct8_1d_internal_c(c, stride, min, max, 0); +} - inv_dct8_1d(in, in_s * 2, tmp, 1, max); +static NOINLINE void +inv_dct16_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, int tx64) +{ + assert(stride > 0); + inv_dct8_1d_internal_c(c, stride << 1, min, max, tx64); + + const int in1 = c[1 * stride], in3 = c[3 * stride]; + const int in5 = c[5 * stride], in7 = c[7 * stride]; - const int in1 = in[ 1 * in_s], in3 = in[ 3 * in_s]; - const int in5 = in[ 5 * in_s], in7 = in[ 7 * in_s]; - const int in9 = in[ 9 * in_s], in11 = in[11 * in_s]; - const int in13 = in[13 * in_s], in15 = in[15 * in_s]; + int t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a; + if (tx64) { + t8a = (in1 * 401 + 2048) >> 12; + t9a = (in7 * -2598 + 2048) >> 12; + t10a = (in5 * 1931 + 2048) >> 12; + t11a = (in3 * -1189 + 2048) >> 12; + t12a = (in3 * 3920 + 2048) >> 12; + t13a = (in5 * 3612 + 2048) >> 12; + t14a = (in7 * 3166 + 2048) >> 12; + t15a = (in1 * 4076 + 2048) >> 12; + } else { + const int in9 = c[ 9 * stride], in11 = c[11 * stride]; + const int in13 = c[13 * stride], in15 = c[15 * stride]; - int t8a = ((in1 * 401 - in15 * (4076 - 4096) + 2048) >> 12) - in15; - int t15a = ((in1 * (4076 - 4096) + in15 * 401 + 2048) >> 12) + in1; - int t9a = (in9 * 1583 - in7 * 1299 + 1024) >> 11; - int t14a = (in9 * 1299 + in7 * 1583 + 1024) >> 11; - int t10a = ((in5 * 1931 - in11 * (3612 - 4096) + 2048) >> 12) - in11; - int t13a = ((in5 * (3612 - 4096) + in11 * 1931 + 2048) >> 12) + in5; - int t11a = ((in13 * (3920 - 4096) - in3 * 1189 + 2048) >> 12) + in13; - int t12a = ((in13 * 1189 + in3 * (3920 - 4096) + 2048) >> 12) + in3; + t8a = ((in1 * 401 - in15 * (4076 - 4096) + 2048) >> 12) - in15; + t9a = (in9 * 1583 - in7 * 1299 + 1024) >> 11; + t10a = ((in5 * 1931 - in11 * (3612 - 4096) + 2048) >> 12) - in11; + t11a = ((in13 * (3920 - 4096) - in3 * 1189 + 2048) >> 12) + in13; + t12a = ((in13 * 1189 + in3 * (3920 - 4096) + 2048) >> 12) + in3; + t13a = ((in5 * (3612 - 4096) + in11 * 1931 + 2048) >> 12) + in5; + t14a = (in9 * 1299 + in7 * 1583 + 1024) >> 11; + t15a = ((in1 * (4076 - 4096) + in15 * 401 + 2048) >> 12) + in1; + } int t8 = CLIP(t8a + t9a); int t9 = CLIP(t8a - t9a); int t10 = CLIP(t11a - t10a); int t11 = CLIP(t11a + t10a); @@ -163,62 +208,97 @@ t10a = ((t13 - t10) * 181 + 128) >> 8; t13a = ((t13 + t10) * 181 + 128) >> 8; t11 = ((t12a - t11a) * 181 + 128) >> 8; t12 = ((t12a + t11a) * 181 + 128) >> 8; - out[ 0 * out_s] = CLIP(tmp[0] + t15a); - out[ 1 * out_s] = CLIP(tmp[1] + t14); - out[ 2 * out_s] = CLIP(tmp[2] + t13a); - out[ 3 * out_s] = CLIP(tmp[3] + t12); - out[ 4 * out_s] = CLIP(tmp[4] + t11); - out[ 5 * out_s] = CLIP(tmp[5] + t10a); - out[ 6 * out_s] = CLIP(tmp[6] + t9); - out[ 7 * out_s] = CLIP(tmp[7] + t8a); - out[ 8 * out_s] = CLIP(tmp[7] - t8a); - out[ 9 * out_s] = CLIP(tmp[6] - t9); - out[10 * out_s] = CLIP(tmp[5] - t10a); - out[11 * out_s] = CLIP(tmp[4] - t11); - out[12 * out_s] = CLIP(tmp[3] - t12); - out[13 * out_s] = CLIP(tmp[2] - t13a); - out[14 * out_s] = CLIP(tmp[1] - t14); - out[15 * out_s] = CLIP(tmp[0] - t15a); + const int t0 = c[ 0 * stride]; + const int t1 = c[ 2 * stride]; + const int t2 = c[ 4 * stride]; + const int t3 = c[ 6 * stride]; + const int t4 = c[ 8 * stride]; + const int t5 = c[10 * stride]; + const int t6 = c[12 * stride]; + const int t7 = c[14 * stride]; + + c[ 0 * stride] = CLIP(t0 + t15a); + c[ 1 * stride] = CLIP(t1 + t14); + c[ 2 * stride] = CLIP(t2 + t13a); + c[ 3 * stride] = CLIP(t3 + t12); + c[ 4 * stride] = CLIP(t4 + t11); + c[ 5 * stride] = CLIP(t5 + t10a); + c[ 6 * stride] = CLIP(t6 + t9); + c[ 7 * stride] = CLIP(t7 + t8a); + c[ 8 * stride] = CLIP(t7 - t8a); + c[ 9 * stride] = CLIP(t6 - t9); + c[10 * stride] = CLIP(t5 - t10a); + c[11 * stride] = CLIP(t4 - t11); + c[12 * stride] = CLIP(t3 - t12); + c[13 * stride] = CLIP(t2 - t13a); + c[14 * stride] = CLIP(t1 - t14); + c[15 * stride] = CLIP(t0 - t15a); +} + +void dav1d_inv_dct16_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + inv_dct16_1d_internal_c(c, stride, min, max, 0); } -static void NOINLINE -inv_dct32_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int max) +static NOINLINE void +inv_dct32_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, const int tx64) { - const int min = -max - 1; - coef tmp[16]; + assert(stride > 0); + inv_dct16_1d_internal_c(c, stride << 1, min, max, tx64); - inv_dct16_1d(in, in_s * 2, tmp, 1, max); + const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; + const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; + const int in9 = c[ 9 * stride], in11 = c[11 * stride]; + const int in13 = c[13 * stride], in15 = c[15 * stride]; - const int in1 = in[ 1 * in_s], in3 = in[ 3 * in_s]; - const int in5 = in[ 5 * in_s], in7 = in[ 7 * in_s]; - const int in9 = in[ 9 * in_s], in11 = in[11 * in_s]; - const int in13 = in[13 * in_s], in15 = in[15 * in_s]; - const int in17 = in[17 * in_s], in19 = in[19 * in_s]; - const int in21 = in[21 * in_s], in23 = in[23 * in_s]; - const int in25 = in[25 * in_s], in27 = in[27 * in_s]; - const int in29 = in[29 * in_s], in31 = in[31 * in_s]; + int t16a, t17a, t18a, t19a, t20a, t21a, t22a, t23a; + int t24a, t25a, t26a, t27a, t28a, t29a, t30a, t31a; + if (tx64) { + t16a = (in1 * 201 + 2048) >> 12; + t17a = (in15 * -2751 + 2048) >> 12; + t18a = (in9 * 1751 + 2048) >> 12; + t19a = (in7 * -1380 + 2048) >> 12; + t20a = (in5 * 995 + 2048) >> 12; + t21a = (in11 * -2106 + 2048) >> 12; + t22a = (in13 * 2440 + 2048) >> 12; + t23a = (in3 * -601 + 2048) >> 12; + t24a = (in3 * 4052 + 2048) >> 12; + t25a = (in13 * 3290 + 2048) >> 12; + t26a = (in11 * 3513 + 2048) >> 12; + t27a = (in5 * 3973 + 2048) >> 12; + t28a = (in7 * 3857 + 2048) >> 12; + t29a = (in9 * 3703 + 2048) >> 12; + t30a = (in15 * 3035 + 2048) >> 12; + t31a = (in1 * 4091 + 2048) >> 12; + } else { + const int in17 = c[17 * stride], in19 = c[19 * stride]; + const int in21 = c[21 * stride], in23 = c[23 * stride]; + const int in25 = c[25 * stride], in27 = c[27 * stride]; + const int in29 = c[29 * stride], in31 = c[31 * stride]; - int t16a = ((in1 * 201 - in31 * (4091 - 4096) + 2048) >> 12) - in31; - int t31a = ((in1 * (4091 - 4096) + in31 * 201 + 2048) >> 12) + in1; - int t17a = ((in17 * (3035 - 4096) - in15 * 2751 + 2048) >> 12) + in17; - int t30a = ((in17 * 2751 + in15 * (3035 - 4096) + 2048) >> 12) + in15; - int t18a = ((in9 * 1751 - in23 * (3703 - 4096) + 2048) >> 12) - in23; - int t29a = ((in9 * (3703 - 4096) + in23 * 1751 + 2048) >> 12) + in9; - int t19a = ((in25 * (3857 - 4096) - in7 * 1380 + 2048) >> 12) + in25; - int t28a = ((in25 * 1380 + in7 * (3857 - 4096) + 2048) >> 12) + in7; - int t20a = ((in5 * 995 - in27 * (3973 - 4096) + 2048) >> 12) - in27; - int t27a = ((in5 * (3973 - 4096) + in27 * 995 + 2048) >> 12) + in5; - int t21a = ((in21 * (3513 - 4096) - in11 * 2106 + 2048) >> 12) + in21; - int t26a = ((in21 * 2106 + in11 * (3513 - 4096) + 2048) >> 12) + in11; - int t22a = (in13 * 1220 - in19 * 1645 + 1024) >> 11; - int t25a = (in13 * 1645 + in19 * 1220 + 1024) >> 11; - int t23a = ((in29 * (4052 - 4096) - in3 * 601 + 2048) >> 12) + in29; - int t24a = ((in29 * 601 + in3 * (4052 - 4096) + 2048) >> 12) + in3; + t16a = ((in1 * 201 - in31 * (4091 - 4096) + 2048) >> 12) - in31; + t17a = ((in17 * (3035 - 4096) - in15 * 2751 + 2048) >> 12) + in17; + t18a = ((in9 * 1751 - in23 * (3703 - 4096) + 2048) >> 12) - in23; + t19a = ((in25 * (3857 - 4096) - in7 * 1380 + 2048) >> 12) + in25; + t20a = ((in5 * 995 - in27 * (3973 - 4096) + 2048) >> 12) - in27; + t21a = ((in21 * (3513 - 4096) - in11 * 2106 + 2048) >> 12) + in21; + t22a = (in13 * 1220 - in19 * 1645 + 1024) >> 11; + t23a = ((in29 * (4052 - 4096) - in3 * 601 + 2048) >> 12) + in29; + t24a = ((in29 * 601 + in3 * (4052 - 4096) + 2048) >> 12) + in3; + t25a = (in13 * 1645 + in19 * 1220 + 1024) >> 11; + t26a = ((in21 * 2106 + in11 * (3513 - 4096) + 2048) >> 12) + in11; + t27a = ((in5 * (3973 - 4096) + in27 * 995 + 2048) >> 12) + in5; + t28a = ((in25 * 1380 + in7 * (3857 - 4096) + 2048) >> 12) + in7; + t29a = ((in9 * (3703 - 4096) + in23 * 1751 + 2048) >> 12) + in9; + t30a = ((in17 * 2751 + in15 * (3035 - 4096) + 2048) >> 12) + in15; + t31a = ((in1 * (4091 - 4096) + in31 * 201 + 2048) >> 12) + in1; + } int t16 = CLIP(t16a + t17a); int t17 = CLIP(t16a - t17a); int t18 = CLIP(t19a - t18a); int t19 = CLIP(t19a + t18a); @@ -294,102 +374,114 @@ t22 = ((t25a - t22a) * 181 + 128) >> 8; t25 = ((t25a + t22a) * 181 + 128) >> 8; t23a = ((t24 - t23 ) * 181 + 128) >> 8; t24a = ((t24 + t23 ) * 181 + 128) >> 8; - out[ 0 * out_s] = CLIP(tmp[ 0] + t31); - out[ 1 * out_s] = CLIP(tmp[ 1] + t30a); - out[ 2 * out_s] = CLIP(tmp[ 2] + t29); - out[ 3 * out_s] = CLIP(tmp[ 3] + t28a); - out[ 4 * out_s] = CLIP(tmp[ 4] + t27); - out[ 5 * out_s] = CLIP(tmp[ 5] + t26a); - out[ 6 * out_s] = CLIP(tmp[ 6] + t25); - out[ 7 * out_s] = CLIP(tmp[ 7] + t24a); - out[ 8 * out_s] = CLIP(tmp[ 8] + t23a); - out[ 9 * out_s] = CLIP(tmp[ 9] + t22); - out[10 * out_s] = CLIP(tmp[10] + t21a); - out[11 * out_s] = CLIP(tmp[11] + t20); - out[12 * out_s] = CLIP(tmp[12] + t19a); - out[13 * out_s] = CLIP(tmp[13] + t18); - out[14 * out_s] = CLIP(tmp[14] + t17a); - out[15 * out_s] = CLIP(tmp[15] + t16); - out[16 * out_s] = CLIP(tmp[15] - t16); - out[17 * out_s] = CLIP(tmp[14] - t17a); - out[18 * out_s] = CLIP(tmp[13] - t18); - out[19 * out_s] = CLIP(tmp[12] - t19a); - out[20 * out_s] = CLIP(tmp[11] - t20); - out[21 * out_s] = CLIP(tmp[10] - t21a); - out[22 * out_s] = CLIP(tmp[ 9] - t22); - out[23 * out_s] = CLIP(tmp[ 8] - t23a); - out[24 * out_s] = CLIP(tmp[ 7] - t24a); - out[25 * out_s] = CLIP(tmp[ 6] - t25); - out[26 * out_s] = CLIP(tmp[ 5] - t26a); - out[27 * out_s] = CLIP(tmp[ 4] - t27); - out[28 * out_s] = CLIP(tmp[ 3] - t28a); - out[29 * out_s] = CLIP(tmp[ 2] - t29); - out[30 * out_s] = CLIP(tmp[ 1] - t30a); - out[31 * out_s] = CLIP(tmp[ 0] - t31); + const int t0 = c[ 0 * stride]; + const int t1 = c[ 2 * stride]; + const int t2 = c[ 4 * stride]; + const int t3 = c[ 6 * stride]; + const int t4 = c[ 8 * stride]; + const int t5 = c[10 * stride]; + const int t6 = c[12 * stride]; + const int t7 = c[14 * stride]; + const int t8 = c[16 * stride]; + const int t9 = c[18 * stride]; + const int t10 = c[20 * stride]; + const int t11 = c[22 * stride]; + const int t12 = c[24 * stride]; + const int t13 = c[26 * stride]; + const int t14 = c[28 * stride]; + const int t15 = c[30 * stride]; + + c[ 0 * stride] = CLIP(t0 + t31); + c[ 1 * stride] = CLIP(t1 + t30a); + c[ 2 * stride] = CLIP(t2 + t29); + c[ 3 * stride] = CLIP(t3 + t28a); + c[ 4 * stride] = CLIP(t4 + t27); + c[ 5 * stride] = CLIP(t5 + t26a); + c[ 6 * stride] = CLIP(t6 + t25); + c[ 7 * stride] = CLIP(t7 + t24a); + c[ 8 * stride] = CLIP(t8 + t23a); + c[ 9 * stride] = CLIP(t9 + t22); + c[10 * stride] = CLIP(t10 + t21a); + c[11 * stride] = CLIP(t11 + t20); + c[12 * stride] = CLIP(t12 + t19a); + c[13 * stride] = CLIP(t13 + t18); + c[14 * stride] = CLIP(t14 + t17a); + c[15 * stride] = CLIP(t15 + t16); + c[16 * stride] = CLIP(t15 - t16); + c[17 * stride] = CLIP(t14 - t17a); + c[18 * stride] = CLIP(t13 - t18); + c[19 * stride] = CLIP(t12 - t19a); + c[20 * stride] = CLIP(t11 - t20); + c[21 * stride] = CLIP(t10 - t21a); + c[22 * stride] = CLIP(t9 - t22); + c[23 * stride] = CLIP(t8 - t23a); + c[24 * stride] = CLIP(t7 - t24a); + c[25 * stride] = CLIP(t6 - t25); + c[26 * stride] = CLIP(t5 - t26a); + c[27 * stride] = CLIP(t4 - t27); + c[28 * stride] = CLIP(t3 - t28a); + c[29 * stride] = CLIP(t2 - t29); + c[30 * stride] = CLIP(t1 - t30a); + c[31 * stride] = CLIP(t0 - t31); } -static void NOINLINE -inv_dct64_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int max) +void dav1d_inv_dct32_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) { - const int min = -max - 1; - coef tmp[32]; - - inv_dct32_1d(in, in_s * 2, tmp, 1, max); + inv_dct32_1d_internal_c(c, stride, min, max, 0); +} - const int in1 = in[ 1 * in_s], in3 = in[ 3 * in_s]; - const int in5 = in[ 5 * in_s], in7 = in[ 7 * in_s]; - const int in9 = in[ 9 * in_s], in11 = in[11 * in_s]; - const int in13 = in[13 * in_s], in15 = in[15 * in_s]; - const int in17 = in[17 * in_s], in19 = in[19 * in_s]; - const int in21 = in[21 * in_s], in23 = in[23 * in_s]; - const int in25 = in[25 * in_s], in27 = in[27 * in_s]; - const int in29 = in[29 * in_s], in31 = in[31 * in_s]; - const int in33 = in[33 * in_s], in35 = in[35 * in_s]; - const int in37 = in[37 * in_s], in39 = in[39 * in_s]; - const int in41 = in[41 * in_s], in43 = in[43 * in_s]; - const int in45 = in[45 * in_s], in47 = in[47 * in_s]; - const int in49 = in[49 * in_s], in51 = in[51 * in_s]; - const int in53 = in[53 * in_s], in55 = in[55 * in_s]; - const int in57 = in[57 * in_s], in59 = in[59 * in_s]; - const int in61 = in[61 * in_s], in63 = in[63 * in_s]; +void dav1d_inv_dct64_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + assert(stride > 0); + inv_dct32_1d_internal_c(c, stride << 1, min, max, 1); + + const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; + const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; + const int in9 = c[ 9 * stride], in11 = c[11 * stride]; + const int in13 = c[13 * stride], in15 = c[15 * stride]; + const int in17 = c[17 * stride], in19 = c[19 * stride]; + const int in21 = c[21 * stride], in23 = c[23 * stride]; + const int in25 = c[25 * stride], in27 = c[27 * stride]; + const int in29 = c[29 * stride], in31 = c[31 * stride]; - int t32a = ((in1 * 101 - in63 * (4095 - 4096) + 2048) >> 12) - in63; - int t33a = ((in33 * (2967 - 4096) - in31 * 2824 + 2048) >> 12) + in33; - int t34a = ((in17 * 1660 - in47 * (3745 - 4096) + 2048) >> 12) - in47; - int t35a = (in49 * 1911 - in15 * 737 + 1024) >> 11; - int t36a = ((in9 * 897 - in55 * (3996 - 4096) + 2048) >> 12) - in55; - int t37a = ((in41 * (3461 - 4096) - in23 * 2191 + 2048) >> 12) + in41; - int t38a = ((in25 * 2359 - in39 * (3349 - 4096) + 2048) >> 12) - in39; - int t39a = (in57 * 2018 - in7 * 350 + 1024) >> 11; - int t40a = ((in5 * 501 - in59 * (4065 - 4096) + 2048) >> 12) - in59; - int t41a = ((in37 * (3229 - 4096) - in27 * 2520 + 2048) >> 12) + in37; - int t42a = ((in21 * 2019 - in43 * (3564 - 4096) + 2048) >> 12) - in43; - int t43a = (in53 * 1974 - in11 * 546 + 1024) >> 11; - int t44a = ((in13 * 1285 - in51 * (3889 - 4096) + 2048) >> 12) - in51; - int t45a = ((in45 * (3659 - 4096) - in19 * 1842 + 2048) >> 12) + in45; - int t46a = ((in29 * 2675 - in35 * (3102 - 4096) + 2048) >> 12) - in35; - int t47a = ((in61 * (4085 - 4096) - in3 * 301 + 2048) >> 12) + in61; - int t48a = ((in61 * 301 + in3 * (4085 - 4096) + 2048) >> 12) + in3; - int t49a = ((in29 * (3102 - 4096) + in35 * 2675 + 2048) >> 12) + in29; - int t50a = ((in45 * 1842 + in19 * (3659 - 4096) + 2048) >> 12) + in19; - int t51a = ((in13 * (3889 - 4096) + in51 * 1285 + 2048) >> 12) + in13; - int t52a = (in53 * 546 + in11 * 1974 + 1024) >> 11; - int t53a = ((in21 * (3564 - 4096) + in43 * 2019 + 2048) >> 12) + in21; - int t54a = ((in37 * 2520 + in27 * (3229 - 4096) + 2048) >> 12) + in27; - int t55a = ((in5 * (4065 - 4096) + in59 * 501 + 2048) >> 12) + in5; - int t56a = (in57 * 350 + in7 * 2018 + 1024) >> 11; - int t57a = ((in25 * (3349 - 4096) + in39 * 2359 + 2048) >> 12) + in25; - int t58a = ((in41 * 2191 + in23 * (3461 - 4096) + 2048) >> 12) + in23; - int t59a = ((in9 * (3996 - 4096) + in55 * 897 + 2048) >> 12) + in9; - int t60a = (in49 * 737 + in15 * 1911 + 1024) >> 11; - int t61a = ((in17 * (3745 - 4096) + in47 * 1660 + 2048) >> 12) + in17; - int t62a = ((in33 * 2824 + in31 * (2967 - 4096) + 2048) >> 12) + in31; - int t63a = ((in1 * (4095 - 4096) + in63 * 101 + 2048) >> 12) + in1; + int t32a = (in1 * 101 + 2048) >> 12; + int t33a = (in31 * -2824 + 2048) >> 12; + int t34a = (in17 * 1660 + 2048) >> 12; + int t35a = (in15 * -1474 + 2048) >> 12; + int t36a = (in9 * 897 + 2048) >> 12; + int t37a = (in23 * -2191 + 2048) >> 12; + int t38a = (in25 * 2359 + 2048) >> 12; + int t39a = (in7 * -700 + 2048) >> 12; + int t40a = (in5 * 501 + 2048) >> 12; + int t41a = (in27 * -2520 + 2048) >> 12; + int t42a = (in21 * 2019 + 2048) >> 12; + int t43a = (in11 * -1092 + 2048) >> 12; + int t44a = (in13 * 1285 + 2048) >> 12; + int t45a = (in19 * -1842 + 2048) >> 12; + int t46a = (in29 * 2675 + 2048) >> 12; + int t47a = (in3 * -301 + 2048) >> 12; + int t48a = (in3 * 4085 + 2048) >> 12; + int t49a = (in29 * 3102 + 2048) >> 12; + int t50a = (in19 * 3659 + 2048) >> 12; + int t51a = (in13 * 3889 + 2048) >> 12; + int t52a = (in11 * 3948 + 2048) >> 12; + int t53a = (in21 * 3564 + 2048) >> 12; + int t54a = (in27 * 3229 + 2048) >> 12; + int t55a = (in5 * 4065 + 2048) >> 12; + int t56a = (in7 * 4036 + 2048) >> 12; + int t57a = (in25 * 3349 + 2048) >> 12; + int t58a = (in23 * 3461 + 2048) >> 12; + int t59a = (in9 * 3996 + 2048) >> 12; + int t60a = (in15 * 3822 + 2048) >> 12; + int t61a = (in17 * 3745 + 2048) >> 12; + int t62a = (in31 * 2967 + 2048) >> 12; + int t63a = (in1 * 4095 + 2048) >> 12; int t32 = CLIP(t32a + t33a); int t33 = CLIP(t32a - t33a); int t34 = CLIP(t35a - t34a); int t35 = CLIP(t35a + t34a); @@ -587,80 +679,115 @@ t52 = ((t43a + t52a) * 181 + 128) >> 8; t53a = ((t42 + t53 ) * 181 + 128) >> 8; t54 = ((t41a + t54a) * 181 + 128) >> 8; t55a = ((t40 + t55 ) * 181 + 128) >> 8; - out[ 0 * out_s] = CLIP(tmp[ 0] + t63a); - out[ 1 * out_s] = CLIP(tmp[ 1] + t62); - out[ 2 * out_s] = CLIP(tmp[ 2] + t61a); - out[ 3 * out_s] = CLIP(tmp[ 3] + t60); - out[ 4 * out_s] = CLIP(tmp[ 4] + t59a); - out[ 5 * out_s] = CLIP(tmp[ 5] + t58); - out[ 6 * out_s] = CLIP(tmp[ 6] + t57a); - out[ 7 * out_s] = CLIP(tmp[ 7] + t56); - out[ 8 * out_s] = CLIP(tmp[ 8] + t55a); - out[ 9 * out_s] = CLIP(tmp[ 9] + t54); - out[10 * out_s] = CLIP(tmp[10] + t53a); - out[11 * out_s] = CLIP(tmp[11] + t52); - out[12 * out_s] = CLIP(tmp[12] + t51a); - out[13 * out_s] = CLIP(tmp[13] + t50); - out[14 * out_s] = CLIP(tmp[14] + t49a); - out[15 * out_s] = CLIP(tmp[15] + t48); - out[16 * out_s] = CLIP(tmp[16] + t47); - out[17 * out_s] = CLIP(tmp[17] + t46a); - out[18 * out_s] = CLIP(tmp[18] + t45); - out[19 * out_s] = CLIP(tmp[19] + t44a); - out[20 * out_s] = CLIP(tmp[20] + t43); - out[21 * out_s] = CLIP(tmp[21] + t42a); - out[22 * out_s] = CLIP(tmp[22] + t41); - out[23 * out_s] = CLIP(tmp[23] + t40a); - out[24 * out_s] = CLIP(tmp[24] + t39); - out[25 * out_s] = CLIP(tmp[25] + t38a); - out[26 * out_s] = CLIP(tmp[26] + t37); - out[27 * out_s] = CLIP(tmp[27] + t36a); - out[28 * out_s] = CLIP(tmp[28] + t35); - out[29 * out_s] = CLIP(tmp[29] + t34a); - out[30 * out_s] = CLIP(tmp[30] + t33); - out[31 * out_s] = CLIP(tmp[31] + t32a); - out[32 * out_s] = CLIP(tmp[31] - t32a); - out[33 * out_s] = CLIP(tmp[30] - t33); - out[34 * out_s] = CLIP(tmp[29] - t34a); - out[35 * out_s] = CLIP(tmp[28] - t35); - out[36 * out_s] = CLIP(tmp[27] - t36a); - out[37 * out_s] = CLIP(tmp[26] - t37); - out[38 * out_s] = CLIP(tmp[25] - t38a); - out[39 * out_s] = CLIP(tmp[24] - t39); - out[40 * out_s] = CLIP(tmp[23] - t40a); - out[41 * out_s] = CLIP(tmp[22] - t41); - out[42 * out_s] = CLIP(tmp[21] - t42a); - out[43 * out_s] = CLIP(tmp[20] - t43); - out[44 * out_s] = CLIP(tmp[19] - t44a); - out[45 * out_s] = CLIP(tmp[18] - t45); - out[46 * out_s] = CLIP(tmp[17] - t46a); - out[47 * out_s] = CLIP(tmp[16] - t47); - out[48 * out_s] = CLIP(tmp[15] - t48); - out[49 * out_s] = CLIP(tmp[14] - t49a); - out[50 * out_s] = CLIP(tmp[13] - t50); - out[51 * out_s] = CLIP(tmp[12] - t51a); - out[52 * out_s] = CLIP(tmp[11] - t52); - out[53 * out_s] = CLIP(tmp[10] - t53a); - out[54 * out_s] = CLIP(tmp[ 9] - t54); - out[55 * out_s] = CLIP(tmp[ 8] - t55a); - out[56 * out_s] = CLIP(tmp[ 7] - t56); - out[57 * out_s] = CLIP(tmp[ 6] - t57a); - out[58 * out_s] = CLIP(tmp[ 5] - t58); - out[59 * out_s] = CLIP(tmp[ 4] - t59a); - out[60 * out_s] = CLIP(tmp[ 3] - t60); - out[61 * out_s] = CLIP(tmp[ 2] - t61a); - out[62 * out_s] = CLIP(tmp[ 1] - t62); - out[63 * out_s] = CLIP(tmp[ 0] - t63a); + const int t0 = c[ 0 * stride]; + const int t1 = c[ 2 * stride]; + const int t2 = c[ 4 * stride]; + const int t3 = c[ 6 * stride]; + const int t4 = c[ 8 * stride]; + const int t5 = c[10 * stride]; + const int t6 = c[12 * stride]; + const int t7 = c[14 * stride]; + const int t8 = c[16 * stride]; + const int t9 = c[18 * stride]; + const int t10 = c[20 * stride]; + const int t11 = c[22 * stride]; + const int t12 = c[24 * stride]; + const int t13 = c[26 * stride]; + const int t14 = c[28 * stride]; + const int t15 = c[30 * stride]; + const int t16 = c[32 * stride]; + const int t17 = c[34 * stride]; + const int t18 = c[36 * stride]; + const int t19 = c[38 * stride]; + const int t20 = c[40 * stride]; + const int t21 = c[42 * stride]; + const int t22 = c[44 * stride]; + const int t23 = c[46 * stride]; + const int t24 = c[48 * stride]; + const int t25 = c[50 * stride]; + const int t26 = c[52 * stride]; + const int t27 = c[54 * stride]; + const int t28 = c[56 * stride]; + const int t29 = c[58 * stride]; + const int t30 = c[60 * stride]; + const int t31 = c[62 * stride]; + + c[ 0 * stride] = CLIP(t0 + t63a); + c[ 1 * stride] = CLIP(t1 + t62); + c[ 2 * stride] = CLIP(t2 + t61a); + c[ 3 * stride] = CLIP(t3 + t60); + c[ 4 * stride] = CLIP(t4 + t59a); + c[ 5 * stride] = CLIP(t5 + t58); + c[ 6 * stride] = CLIP(t6 + t57a); + c[ 7 * stride] = CLIP(t7 + t56); + c[ 8 * stride] = CLIP(t8 + t55a); + c[ 9 * stride] = CLIP(t9 + t54); + c[10 * stride] = CLIP(t10 + t53a); + c[11 * stride] = CLIP(t11 + t52); + c[12 * stride] = CLIP(t12 + t51a); + c[13 * stride] = CLIP(t13 + t50); + c[14 * stride] = CLIP(t14 + t49a); + c[15 * stride] = CLIP(t15 + t48); + c[16 * stride] = CLIP(t16 + t47); + c[17 * stride] = CLIP(t17 + t46a); + c[18 * stride] = CLIP(t18 + t45); + c[19 * stride] = CLIP(t19 + t44a); + c[20 * stride] = CLIP(t20 + t43); + c[21 * stride] = CLIP(t21 + t42a); + c[22 * stride] = CLIP(t22 + t41); + c[23 * stride] = CLIP(t23 + t40a); + c[24 * stride] = CLIP(t24 + t39); + c[25 * stride] = CLIP(t25 + t38a); + c[26 * stride] = CLIP(t26 + t37); + c[27 * stride] = CLIP(t27 + t36a); + c[28 * stride] = CLIP(t28 + t35); + c[29 * stride] = CLIP(t29 + t34a); + c[30 * stride] = CLIP(t30 + t33); + c[31 * stride] = CLIP(t31 + t32a); + c[32 * stride] = CLIP(t31 - t32a); + c[33 * stride] = CLIP(t30 - t33); + c[34 * stride] = CLIP(t29 - t34a); + c[35 * stride] = CLIP(t28 - t35); + c[36 * stride] = CLIP(t27 - t36a); + c[37 * stride] = CLIP(t26 - t37); + c[38 * stride] = CLIP(t25 - t38a); + c[39 * stride] = CLIP(t24 - t39); + c[40 * stride] = CLIP(t23 - t40a); + c[41 * stride] = CLIP(t22 - t41); + c[42 * stride] = CLIP(t21 - t42a); + c[43 * stride] = CLIP(t20 - t43); + c[44 * stride] = CLIP(t19 - t44a); + c[45 * stride] = CLIP(t18 - t45); + c[46 * stride] = CLIP(t17 - t46a); + c[47 * stride] = CLIP(t16 - t47); + c[48 * stride] = CLIP(t15 - t48); + c[49 * stride] = CLIP(t14 - t49a); + c[50 * stride] = CLIP(t13 - t50); + c[51 * stride] = CLIP(t12 - t51a); + c[52 * stride] = CLIP(t11 - t52); + c[53 * stride] = CLIP(t10 - t53a); + c[54 * stride] = CLIP(t9 - t54); + c[55 * stride] = CLIP(t8 - t55a); + c[56 * stride] = CLIP(t7 - t56); + c[57 * stride] = CLIP(t6 - t57a); + c[58 * stride] = CLIP(t5 - t58); + c[59 * stride] = CLIP(t4 - t59a); + c[60 * stride] = CLIP(t3 - t60); + c[61 * stride] = CLIP(t2 - t61a); + c[62 * stride] = CLIP(t1 - t62); + c[63 * stride] = CLIP(t0 - t63a); } -static void NOINLINE -inv_adst4_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int range) +static NOINLINE void +inv_adst4_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s, + const int min, const int max, + int32_t *const out, const ptrdiff_t out_s) { + assert(in_s > 0 && out_s != 0); const int in0 = in[0 * in_s], in1 = in[1 * in_s]; const int in2 = in[2 * in_s], in3 = in[3 * in_s]; out[0 * out_s] = (( 1321 * in0 + (3803 - 4096) * in2 + (2482 - 4096) * in3 + (3344 - 4096) * in1 + 2048) >> 12) + @@ -672,64 +799,65 @@ out[3 * out_s] = (((3803 - 4096) * in0 + (2482 - 4096) * in2 - 1321 * in3 - (3344 - 4096) * in1 + 2048) >> 12) + in0 + in2 - in1; } -static void NOINLINE -inv_adst8_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int max) +static NOINLINE void +inv_adst8_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s, + const int min, const int max, + int32_t *const out, const ptrdiff_t out_s) { - const int min = -max - 1; + assert(in_s > 0 && out_s != 0); const int in0 = in[0 * in_s], in1 = in[1 * in_s]; const int in2 = in[2 * in_s], in3 = in[3 * in_s]; const int in4 = in[4 * in_s], in5 = in[5 * in_s]; const int in6 = in[6 * in_s], in7 = in[7 * in_s]; - int t0a = (((4076 - 4096) * in7 + 401 * in0 + 2048) >> 12) + in7; - int t1a = (( 401 * in7 - (4076 - 4096) * in0 + 2048) >> 12) - in0; - int t2a = (((3612 - 4096) * in5 + 1931 * in2 + 2048) >> 12) + in5; - int t3a = (( 1931 * in5 - (3612 - 4096) * in2 + 2048) >> 12) - in2; - int t4a = ( 1299 * in3 + 1583 * in4 + 1024) >> 11; - int t5a = ( 1583 * in3 - 1299 * in4 + 1024) >> 11; - int t6a = (( 1189 * in1 + (3920 - 4096) * in6 + 2048) >> 12) + in6; - int t7a = (((3920 - 4096) * in1 - 1189 * in6 + 2048) >> 12) + in1; + const int t0a = (((4076 - 4096) * in7 + 401 * in0 + 2048) >> 12) + in7; + const int t1a = (( 401 * in7 - (4076 - 4096) * in0 + 2048) >> 12) - in0; + const int t2a = (((3612 - 4096) * in5 + 1931 * in2 + 2048) >> 12) + in5; + const int t3a = (( 1931 * in5 - (3612 - 4096) * in2 + 2048) >> 12) - in2; + int t4a = ( 1299 * in3 + 1583 * in4 + 1024) >> 11; + int t5a = ( 1583 * in3 - 1299 * in4 + 1024) >> 11; + int t6a = (( 1189 * in1 + (3920 - 4096) * in6 + 2048) >> 12) + in6; + int t7a = (((3920 - 4096) * in1 - 1189 * in6 + 2048) >> 12) + in1; - int t0 = CLIP(t0a + t4a); - int t1 = CLIP(t1a + t5a); - int t2 = CLIP(t2a + t6a); - int t3 = CLIP(t3a + t7a); - int t4 = CLIP(t0a - t4a); - int t5 = CLIP(t1a - t5a); - int t6 = CLIP(t2a - t6a); - int t7 = CLIP(t3a - t7a); + const int t0 = CLIP(t0a + t4a); + const int t1 = CLIP(t1a + t5a); + int t2 = CLIP(t2a + t6a); + int t3 = CLIP(t3a + t7a); + const int t4 = CLIP(t0a - t4a); + const int t5 = CLIP(t1a - t5a); + int t6 = CLIP(t2a - t6a); + int t7 = CLIP(t3a - t7a); t4a = (((3784 - 4096) * t4 + 1567 * t5 + 2048) >> 12) + t4; t5a = (( 1567 * t4 - (3784 - 4096) * t5 + 2048) >> 12) - t5; t6a = (((3784 - 4096) * t7 - 1567 * t6 + 2048) >> 12) + t7; t7a = (( 1567 * t7 + (3784 - 4096) * t6 + 2048) >> 12) + t6; - out[0 * out_s] = CLIP( t0 + t2); - out[7 * out_s] = CLIP(-(t1 + t3)); - t2 = CLIP( t0 - t2); - t3 = CLIP( t1 - t3); - - out[1 * out_s] = CLIP(-(t4a + t6a)); - out[6 * out_s] = CLIP( t5a + t7a ); - t6 = CLIP( t4a - t6a ); - t7 = CLIP( t5a - t7a ); + out[0 * out_s] = CLIP(t0 + t2 ); + out[7 * out_s] = -CLIP(t1 + t3 ); + t2 = CLIP(t0 - t2 ); + t3 = CLIP(t1 - t3 ); + out[1 * out_s] = -CLIP(t4a + t6a); + out[6 * out_s] = CLIP(t5a + t7a); + t6 = CLIP(t4a - t6a); + t7 = CLIP(t5a - t7a); out[3 * out_s] = -(((t2 + t3) * 181 + 128) >> 8); out[4 * out_s] = ((t2 - t3) * 181 + 128) >> 8; out[2 * out_s] = ((t6 + t7) * 181 + 128) >> 8; out[5 * out_s] = -(((t6 - t7) * 181 + 128) >> 8); } -static void NOINLINE -inv_adst16_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int max) +static NOINLINE void +inv_adst16_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s, + const int min, const int max, + int32_t *const out, const ptrdiff_t out_s) { - const int min = -max - 1; + assert(in_s > 0 && out_s != 0); const int in0 = in[ 0 * in_s], in1 = in[ 1 * in_s]; const int in2 = in[ 2 * in_s], in3 = in[ 3 * in_s]; const int in4 = in[ 4 * in_s], in5 = in[ 5 * in_s]; const int in6 = in[ 6 * in_s], in7 = in[ 7 * in_s]; const int in8 = in[ 8 * in_s], in9 = in[ 9 * in_s]; @@ -804,26 +932,26 @@ t12 = ((t12a * (3784 - 4096) + t13a * 1567 + 2048) >> 12) + t12a; t13 = ((t12a * 1567 - t13a * (3784 - 4096) + 2048) >> 12) - t13a; t14 = ((t15a * (3784 - 4096) - t14a * 1567 + 2048) >> 12) + t15a; t15 = ((t15a * 1567 + t14a * (3784 - 4096) + 2048) >> 12) + t14a; - out[ 0 * out_s] = CLIP( t0 + t2 ); - out[15 * out_s] = CLIP(-(t1 + t3) ); - t2a = CLIP( t0 - t2 ); - t3a = CLIP( t1 - t3 ); - out[ 3 * out_s] = CLIP(-(t4a + t6a) ); - out[12 * out_s] = CLIP( t5a + t7a ); - t6 = CLIP( t4a - t6a ); - t7 = CLIP( t5a - t7a ); - out[ 1 * out_s] = CLIP(-(t8a + t10a)); - out[14 * out_s] = CLIP( t9a + t11a ); - t10 = CLIP( t8a - t10a ); - t11 = CLIP( t9a - t11a ); - out[ 2 * out_s] = CLIP( t12 + t14 ); - out[13 * out_s] = CLIP(-(t13 + t15) ); - t14a = CLIP( t12 - t14 ); - t15a = CLIP( t13 - t15 ); + out[ 0 * out_s] = CLIP(t0 + t2 ); + out[15 * out_s] = -CLIP(t1 + t3 ); + t2a = CLIP(t0 - t2 ); + t3a = CLIP(t1 - t3 ); + out[ 3 * out_s] = -CLIP(t4a + t6a ); + out[12 * out_s] = CLIP(t5a + t7a ); + t6 = CLIP(t4a - t6a ); + t7 = CLIP(t5a - t7a ); + out[ 1 * out_s] = -CLIP(t8a + t10a); + out[14 * out_s] = CLIP(t9a + t11a); + t10 = CLIP(t8a - t10a); + t11 = CLIP(t9a - t11a); + out[ 2 * out_s] = CLIP(t12 + t14 ); + out[13 * out_s] = -CLIP(t13 + t15 ); + t14a = CLIP(t12 - t14 ); + t15a = CLIP(t13 - t15 ); out[ 7 * out_s] = -(((t2a + t3a) * 181 + 128) >> 8); out[ 8 * out_s] = ((t2a - t3a) * 181 + 128) >> 8; out[ 4 * out_s] = ((t6 + t7) * 181 + 128) >> 8; out[11 * out_s] = -(((t6 - t7) * 181 + 128) >> 8); @@ -831,69 +959,76 @@ out[ 9 * out_s] = -(((t10 - t11) * 181 + 128) >> 8); out[ 5 * out_s] = -(((t14a + t15a) * 181 + 128) >> 8); out[10 * out_s] = ((t14a - t15a) * 181 + 128) >> 8; } -#define flip_inv_adst(sz) \ -static void inv_flipadst##sz##_1d(const coef *const in, const ptrdiff_t in_s, \ - coef *const out, const ptrdiff_t out_s, const int range) \ +#define inv_adst_1d(sz) \ +void dav1d_inv_adst##sz##_1d_c(int32_t *const c, const ptrdiff_t stride, \ + const int min, const int max) \ { \ - inv_adst##sz##_1d(in, in_s, &out[(sz - 1) * out_s], -out_s, range); \ + inv_adst##sz##_1d_internal_c(c, stride, min, max, c, stride); \ +} \ +void dav1d_inv_flipadst##sz##_1d_c(int32_t *const c, const ptrdiff_t stride, \ + const int min, const int max) \ +{ \ + inv_adst##sz##_1d_internal_c(c, stride, min, max, \ + &c[(sz - 1) * stride], -stride); \ } -flip_inv_adst(4) -flip_inv_adst(8) -flip_inv_adst(16) +inv_adst_1d( 4) +inv_adst_1d( 8) +inv_adst_1d(16) -#undef flip_inv_adst +#undef inv_adst_1d -static void NOINLINE -inv_identity4_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int range) +void dav1d_inv_identity4_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) { - for (int i = 0; i < 4; i++) - out[out_s * i] = in[in_s * i] + ((in[in_s * i] * 1697 + 2048) >> 12); + assert(stride > 0); + for (int i = 0; i < 4; i++) { + const int in = c[stride * i]; + c[stride * i] = in + ((in * 1697 + 2048) >> 12); + } } -static void NOINLINE -inv_identity8_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int range) +void dav1d_inv_identity8_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) { + assert(stride > 0); for (int i = 0; i < 8; i++) - out[out_s * i] = in[in_s * i] * 2; -} - -static void NOINLINE -inv_identity16_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int range) -{ - for (int i = 0; i < 16; i++) - out[out_s * i] = 2 * in[in_s * i] + ((in[in_s * i] * 1697 + 1024) >> 11); + c[stride * i] *= 2; } -static void NOINLINE -inv_identity32_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, const int range) +void dav1d_inv_identity16_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) { - for (int i = 0; i < 32; i++) - out[out_s * i] = in[in_s * i] * 4; + assert(stride > 0); + for (int i = 0; i < 16; i++) { + const int in = c[stride * i]; + c[stride * i] = 2 * in + ((in * 1697 + 1024) >> 11); + } } -static void NOINLINE -inv_wht4_1d(const coef *const in, const ptrdiff_t in_s, - coef *const out, const ptrdiff_t out_s, - const int pass) +void dav1d_inv_identity32_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) { - const int sh = 2 * !pass; - const int in0 = in[0 * in_s] >> sh, in1 = in[1 * in_s] >> sh; - const int in2 = in[2 * in_s] >> sh, in3 = in[3 * in_s] >> sh; + assert(stride > 0); + for (int i = 0; i < 32; i++) + c[stride * i] *= 4; +} + +void dav1d_inv_wht4_1d_c(int32_t *const c, const ptrdiff_t stride) { + assert(stride > 0); + const int in0 = c[0 * stride], in1 = c[1 * stride]; + const int in2 = c[2 * stride], in3 = c[3 * stride]; + const int t0 = in0 + in1; const int t2 = in2 - in3; const int t4 = (t0 - t2) >> 1; const int t3 = t4 - in3; const int t1 = t4 - in1; - out[0 * out_s] = t0 - t3; - out[1 * out_s] = t3; - out[2 * out_s] = t1; - out[3 * out_s] = t2 + t1; + c[0 * stride] = t0 - t3; + c[1 * stride] = t3; + c[2 * stride] = t1; + c[3 * stride] = t2 + t1; } diff --git a/third_party/dav1d/src/itx_tmpl.c b/third_party/dav1d/src/itx_tmpl.c --- a/third_party/dav1d/src/itx_tmpl.c +++ b/third_party/dav1d/src/itx_tmpl.c @@ -1,8 +1,8 @@ /* - * Copyright © 2018, VideoLAN and dav1d authors - * Copyright © 2018, Two Orioles, LLC + * Copyright © 2018-2019, VideoLAN and dav1d authors + * Copyright © 2018-2019, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * @@ -33,82 +33,72 @@ #include "common/attributes.h" #include "common/intops.h" #include "src/itx.h" - -#include "src/itx_1d.c" +#include "src/itx_1d.h" -typedef void (*itx_1d_fn)(const coef *in, ptrdiff_t in_s, - coef *out, ptrdiff_t out_s, const int range); - -static void NOINLINE -inv_txfm_add_c(pixel *dst, const ptrdiff_t stride, - coef *const coeff, const int eob, - const int w, const int h, const int shift, +static NOINLINE void +inv_txfm_add_c(pixel *dst, const ptrdiff_t stride, coef *const coeff, + const int eob, const int w, const int h, const int shift, const itx_1d_fn first_1d_fn, const itx_1d_fn second_1d_fn, const int has_dconly HIGHBD_DECL_SUFFIX) { - int i, j; - assert((h >= 4 && h <= 64) && (w >= 4 && w <= 64)); + assert(w >= 4 && w <= 64); + assert(h >= 4 && h <= 64); + assert(eob >= 0); + const int is_rect2 = w * 2 == h || h * 2 == w; - const int bitdepth = bitdepth_from_max(bitdepth_max); const int rnd = (1 << shift) >> 1; - if (has_dconly && eob == 0) { + if (eob < has_dconly) { int dc = coeff[0]; coeff[0] = 0; if (is_rect2) - dc = (dc * 2896 + 2048) >> 12; - dc = (dc * 2896 + 2048) >> 12; + dc = (dc * 181 + 128) >> 8; + dc = (dc * 181 + 128) >> 8; dc = (dc + rnd) >> shift; - dc = (dc * 2896 + 2048) >> 12; - dc = (dc + 8) >> 4; - for (j = 0; j < h; j++) - for (i = 0; i < w; i++) - dst[i + j * PXSTRIDE(stride)] = - iclip_pixel(dst[i + j * PXSTRIDE(stride)] + dc); + dc = (dc * 181 + 128 + 2048) >> 12; + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel(dst[x] + dc); return; } - assert(eob > 0 || (eob == 0 && !has_dconly)); - - const ptrdiff_t sh = imin(h, 32), sw = imin(w, 32); - // Maximum value for h and w is 64 - coef tmp[4096 /* w * h */], out[64 /* h */], in_mem[64 /* w */]; - const int row_clip_max = (1 << (bitdepth + 8 - 1)) - 1; - const int col_clip_max = (1 << (imax(bitdepth + 6, 16) - 1)) -1; - if (w != sw) memset(&in_mem[sw], 0, (w - sw) * sizeof(*in_mem)); - for (i = 0; i < sh; i++) { - if (w != sw || is_rect2) { - for (j = 0; j < sw; j++) { - in_mem[j] = coeff[i + j * sh]; - if (is_rect2) - in_mem[j] = (in_mem[j] * 2896 + 2048) >> 12; - } - first_1d_fn(in_mem, 1, &tmp[i * w], 1, row_clip_max); - } else { - first_1d_fn(&coeff[i], sh, &tmp[i * w], 1, row_clip_max); - } - for (j = 0; j < w; j++) + const int sh = imin(h, 32), sw = imin(w, 32); #if BITDEPTH == 8 - tmp[i * w + j] = (tmp[i * w + j] + rnd) >> shift; + const int row_clip_min = INT16_MIN; + const int col_clip_min = INT16_MIN; #else - tmp[i * w + j] = iclip((tmp[i * w + j] + rnd) >> shift, - -col_clip_max - 1, col_clip_max); + const int row_clip_min = (int) ((unsigned) ~bitdepth_max << 7); + const int col_clip_min = (int) ((unsigned) ~bitdepth_max << 5); #endif + const int row_clip_max = ~row_clip_min; + const int col_clip_max = ~col_clip_min; + + int32_t tmp[64 * 64], *c = tmp; + for (int y = 0; y < sh; y++, c += w) { + if (is_rect2) + for (int x = 0; x < sw; x++) + c[x] = (coeff[y + x * sh] * 181 + 128) >> 8; + else + for (int x = 0; x < sw; x++) + c[x] = coeff[y + x * sh]; + first_1d_fn(c, 1, row_clip_min, row_clip_max); } - if (h != sh) memset(&tmp[sh * w], 0, w * (h - sh) * sizeof(*tmp)); - for (i = 0; i < w; i++) { - second_1d_fn(&tmp[i], w, out, 1, col_clip_max); - for (j = 0; j < h; j++) - dst[i + j * PXSTRIDE(stride)] = - iclip_pixel(dst[i + j * PXSTRIDE(stride)] + - ((out[j] + 8) >> 4)); - } - memset(coeff, 0, sizeof(*coeff) * sh * sw); + memset(coeff, 0, sizeof(*coeff) * sw * sh); + for (int i = 0; i < w * sh; i++) + tmp[i] = iclip((tmp[i] + rnd) >> shift, col_clip_min, col_clip_max); + + for (int x = 0; x < w; x++) + second_1d_fn(&tmp[x], w, col_clip_min, col_clip_max); + + c = tmp; + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel(dst[x] + ((*c++ + 8) >> 4)); } #define inv_txfm_fn(type1, type2, w, h, shift, has_dconly) \ static void \ inv_txfm_add_##type1##_##type2##_##w##x##h##_c(pixel *dst, \ @@ -116,12 +106,12 @@ coef *const coeff, \ const int eob \ HIGHBD_DECL_SUFFIX) \ { \ inv_txfm_add_c(dst, stride, coeff, eob, w, h, shift, \ - inv_##type1##w##_1d, inv_##type2##h##_1d, has_dconly \ - HIGHBD_TAIL_SUFFIX); \ + dav1d_inv_##type1##w##_1d_c, dav1d_inv_##type2##h##_1d_c, \ + has_dconly HIGHBD_TAIL_SUFFIX); \ } #define inv_txfm_fn64(w, h, shift) \ inv_txfm_fn(dct, dct, w, h, shift, 1) @@ -171,27 +161,25 @@ static void inv_txfm_add_wht_wht_4x4_c(pixel *dst, const ptrdiff_t stride, coef *const coeff, const int eob HIGHBD_DECL_SUFFIX) { - const int bitdepth = bitdepth_from_max(bitdepth_max); - const int col_clip_max = (1 << (imax(bitdepth + 6, 16) - 1)) -1; - const int col_clip_min = -col_clip_max - 1; - coef tmp[4 * 4], out[4]; - - for (int i = 0; i < 4; i++) - inv_wht4_1d(&coeff[i], 4, &tmp[i * 4], 1, 0); - for (int k = 0; k < 4 * 4; k++) - tmp[k] = iclip(tmp[k], col_clip_min, col_clip_max); - - for (int i = 0; i < 4; i++) { - inv_wht4_1d(&tmp[i], 4, out, 1, 1); - for (int j = 0; j < 4; j++) - dst[i + j * PXSTRIDE(stride)] = - iclip_pixel(dst[i + j * PXSTRIDE(stride)] + out[j]); + int32_t tmp[4 * 4], *c = tmp; + for (int y = 0; y < 4; y++, c += 4) { + for (int x = 0; x < 4; x++) + c[x] = coeff[y + x * 4] >> 2; + dav1d_inv_wht4_1d_c(c, 1); } memset(coeff, 0, sizeof(*coeff) * 4 * 4); + + for (int x = 0; x < 4; x++) + dav1d_inv_wht4_1d_c(&tmp[x], 4); + + c = tmp; + for (int y = 0; y < 4; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < 4; x++) + dst[x] = iclip_pixel(dst[x] + *c++); } COLD void bitfn(dav1d_itx_dsp_init)(Dav1dInvTxfmDSPContext *const c) { #define assign_itx_all_fn64(w, h, pfx) \ c->itxfm_add[pfx##TX_##w##X##h][DCT_DCT ] = \ diff --git a/third_party/dav1d/src/levels.h b/third_party/dav1d/src/levels.h --- a/third_party/dav1d/src/levels.h +++ b/third_party/dav1d/src/levels.h @@ -241,12 +241,15 @@ INTER_INTRA_NONE, INTER_INTRA_BLEND, INTER_INTRA_WEDGE, }; -typedef struct mv { - int16_t y, x; +typedef union mv { + struct { + int16_t y, x; + }; + uint32_t n; } mv; enum MotionMode { MM_TRANSLATION, MM_OBMC, @@ -262,16 +265,24 @@ struct { uint8_t y_mode, uv_mode, tx, pal_sz[2]; int8_t y_angle, uv_angle, cfl_alpha[2]; }; // intra struct { + union { + struct { + union mv mv[2]; + uint8_t wedge_idx, mask_sign, interintra_mode; + }; + struct { + union mv mv2d; + int16_t matrix[4]; + }; + }; + uint8_t comp_type, inter_mode, motion_mode, drl_idx; int8_t ref[2]; - uint8_t comp_type, wedge_idx, mask_sign, inter_mode, drl_idx; - uint8_t interintra_type, interintra_mode, motion_mode; - uint8_t max_ytx, filter2d; - uint16_t tx_split[2]; - mv mv[2]; + uint8_t max_ytx, filter2d, interintra_type, tx_split0; + uint16_t tx_split1; }; // inter }; } Av1Block; #endif /* DAV1D_SRC_LEVELS_H */ diff --git a/third_party/dav1d/src/lf_mask.h b/third_party/dav1d/src/lf_mask.h --- a/third_party/dav1d/src/lf_mask.h +++ b/third_party/dav1d/src/lf_mask.h @@ -70,12 +70,12 @@ uint8_t *ly, uint8_t *auv, uint8_t *luv); void dav1d_create_lf_mask_inter(Av1Filter *lflvl, uint8_t (*level_cache)[4], const ptrdiff_t b4_stride, const uint8_t (*level)[8][2], int bx, int by, int iw, int ih, int skip_inter, - enum BlockSize bs, const uint16_t *tx_mask, - enum RectTxfmSize uvtx, + enum BlockSize bs, enum RectTxfmSize max_ytx, + const uint16_t *tx_mask, enum RectTxfmSize uvtx, enum Dav1dPixelLayout layout, uint8_t *ay, uint8_t *ly, uint8_t *auv, uint8_t *luv); void dav1d_calc_eih(Av1FilterLUT *lim_lut, int filter_sharpness); void dav1d_calc_lf_values(uint8_t (*values)[4][8][2], const Dav1dFrameHeader *hdr, const int8_t lf_delta[4]); diff --git a/third_party/dav1d/src/lf_mask.c b/third_party/dav1d/src/lf_mask.c --- a/third_party/dav1d/src/lf_mask.c +++ b/third_party/dav1d/src/lf_mask.c @@ -41,12 +41,12 @@ const int depth, const int y_off, const int x_off, const uint16_t *const tx_masks) { const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[from]; - const int is_split = - depth > 1 ? 0 : (tx_masks[depth] >> (y_off * 4 + x_off)) & 1; + const int is_split = (from == (int) TX_4X4 || depth > 1) ? 0 : + (tx_masks[depth] >> (y_off * 4 + x_off)) & 1; if (is_split) { const enum RectTxfmSize sub = t_dim->sub; const int htw4 = t_dim->w >> 1, hth4 = t_dim->h >> 1; @@ -348,10 +348,11 @@ const ptrdiff_t b4_stride, const uint8_t (*filter_level)[8][2], const int bx, const int by, const int iw, const int ih, const int skip, const enum BlockSize bs, + const enum RectTxfmSize max_ytx, const uint16_t *const tx_masks, const enum RectTxfmSize uvtx, const enum Dav1dPixelLayout layout, uint8_t *const ay, uint8_t *const ly, uint8_t *const auv, uint8_t *const luv) @@ -371,11 +372,11 @@ } level_cache_ptr += b4_stride; } mask_edges_inter(lflvl->filter_y, by4, bx4, bw4, bh4, skip, - dav1d_max_txfm_size_for_bs[bs][0], tx_masks, ay, ly); + max_ytx, tx_masks, ay, ly); } if (!auv) return; const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; diff --git a/third_party/dav1d/src/lib.c b/third_party/dav1d/src/lib.c --- a/third_party/dav1d/src/lib.c +++ b/third_party/dav1d/src/lib.c @@ -29,30 +29,36 @@ #include "vcs_version.h" #include #include +#if defined(__linux__) && defined(HAVE_DLSYM) +#include +#endif + #include "dav1d/dav1d.h" #include "dav1d/data.h" #include "common/mem.h" #include "common/validate.h" +#include "src/cpu.h" #include "src/fg_apply.h" #include "src/internal.h" #include "src/log.h" #include "src/obu.h" #include "src/qm.h" #include "src/ref.h" #include "src/thread_task.h" #include "src/wedge.h" static COLD void init_internal(void) { - dav1d_init_wedge_masks(); + dav1d_init_cpu(); dav1d_init_interintra_masks(); dav1d_init_qm_tables(); dav1d_init_thread(); + dav1d_init_wedge_masks(); } COLD const char *dav1d_version(void) { return DAV1D_VERSION; } @@ -71,10 +77,26 @@ s->frame_size_limit = 0; } static void close_internal(Dav1dContext **const c_out, int flush); +NO_SANITIZE("cfi-icall") // CFI is broken with dlsym() +static COLD size_t get_stack_size_internal(const pthread_attr_t *const thread_attr) { +#if defined(__linux__) && defined(HAVE_DLSYM) && defined(__GLIBC__) + /* glibc has an issue where the size of the TLS is subtracted from the stack + * size instead of allocated separately. As a result the specified stack + * size may be insufficient when used in an application with large amounts + * of TLS data. The following is a workaround to compensate for that. + * See https://sourceware.org/bugzilla/show_bug.cgi?id=11787 */ + size_t (*const get_minstack)(const pthread_attr_t*) = + dlsym(RTLD_DEFAULT, "__pthread_get_minstack"); + if (get_minstack) + return get_minstack(thread_attr) - PTHREAD_STACK_MIN; +#endif + return 0; +} + COLD int dav1d_open(Dav1dContext **const c_out, const Dav1dSettings *const s) { static pthread_once_t initted = PTHREAD_ONCE_INIT; pthread_once(&initted, init_internal); validate_input_or_ret(c_out != NULL, DAV1D_ERR(EINVAL)); @@ -90,11 +112,13 @@ validate_input_or_ret(s->operating_point >= 0 && s->operating_point <= 31, DAV1D_ERR(EINVAL)); pthread_attr_t thread_attr; if (pthread_attr_init(&thread_attr)) return DAV1D_ERR(ENOMEM); - pthread_attr_setstacksize(&thread_attr, 1024 * 1024); + size_t stack_size = 1024 * 1024 + get_stack_size_internal(&thread_attr); + + pthread_attr_setstacksize(&thread_attr, stack_size); Dav1dContext *const c = *c_out = dav1d_alloc_aligned(sizeof(*c), 32); if (!c) goto error; memset(c, 0, sizeof(*c)); @@ -122,21 +146,19 @@ c->fc = dav1d_alloc_aligned(sizeof(*c->fc) * s->n_frame_threads, 32); if (!c->fc) goto error; memset(c->fc, 0, sizeof(*c->fc) * s->n_frame_threads); if (c->n_fc > 1) { c->frame_thread.out_delayed = - malloc(sizeof(*c->frame_thread.out_delayed) * c->n_fc); + calloc(c->n_fc, sizeof(*c->frame_thread.out_delayed)); if (!c->frame_thread.out_delayed) goto error; - memset(c->frame_thread.out_delayed, 0, - sizeof(*c->frame_thread.out_delayed) * c->n_fc); } for (int n = 0; n < s->n_frame_threads; n++) { Dav1dFrameContext *const f = &c->fc[n]; f->c = c; f->lf.last_sharpness = -1; f->n_tc = s->n_tile_threads; - f->tc = dav1d_alloc_aligned(sizeof(*f->tc) * s->n_tile_threads, 32); + f->tc = dav1d_alloc_aligned(sizeof(*f->tc) * s->n_tile_threads, 64); if (!f->tc) goto error; memset(f->tc, 0, sizeof(*f->tc) * s->n_tile_threads); if (f->n_tc > 1) { if (pthread_mutex_init(&f->tile_thread.lock, NULL)) goto error; if (pthread_cond_init(&f->tile_thread.cond, NULL)) { @@ -167,12 +189,11 @@ goto error; } t->tile_thread.td.inited = 1; } } - f->libaom_cm = dav1d_alloc_ref_mv_common(); - if (!f->libaom_cm) goto error; + dav1d_refmvs_init(&f->rf); if (c->n_fc > 1) { if (pthread_mutex_init(&f->frame_thread.td.lock, NULL)) goto error; if (pthread_cond_init(&f->frame_thread.td.cond, NULL)) { pthread_mutex_destroy(&f->frame_thread.td.lock); goto error; @@ -249,25 +270,10 @@ dav1d_close(&c); return res; } -int dav1d_send_data(Dav1dContext *const c, Dav1dData *const in) -{ - validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); - validate_input_or_ret(in != NULL, DAV1D_ERR(EINVAL)); - validate_input_or_ret(in->data == NULL || in->sz, DAV1D_ERR(EINVAL)); - - c->drain = 0; - - if (c->in.data) - return DAV1D_ERR(EAGAIN); - dav1d_data_move_ref(&c->in, in); - - return 0; -} - static int output_image(Dav1dContext *const c, Dav1dPicture *const out, Dav1dPicture *const in) { const Dav1dFilmGrainData *fgdata = &in->frame_hdr->film_grain.data; int has_grain = fgdata->num_y_points || fgdata->num_uv_points[0] || @@ -350,25 +356,17 @@ } while (++drain_count < c->n_fc); return DAV1D_ERR(EAGAIN); } -int dav1d_get_picture(Dav1dContext *const c, Dav1dPicture *const out) +static int gen_picture(Dav1dContext *const c) { int res; - - validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); - validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + Dav1dData *const in = &c->in; - const int drain = c->drain; - c->drain = 1; - - Dav1dData *const in = &c->in; - if (!in->data) { - if (c->n_fc == 1) return DAV1D_ERR(EAGAIN); - return drain_picture(c, out); - } + if (output_picture_ready(c)) + return 0; while (in->sz > 0) { res = dav1d_parse_obus(c, in, 0); if (res < 0) { dav1d_data_unref_internal(in); @@ -382,10 +380,44 @@ break; if (res < 0) return res; } + return 0; +} + +int dav1d_send_data(Dav1dContext *const c, Dav1dData *const in) +{ + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(in != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(in->data == NULL || in->sz, DAV1D_ERR(EINVAL)); + + if (in->data) + c->drain = 0; + if (c->in.data) + return DAV1D_ERR(EAGAIN); + dav1d_data_ref(&c->in, in); + + int res = gen_picture(c); + if (!res) + dav1d_data_unref_internal(in); + + return res; +} + +int dav1d_get_picture(Dav1dContext *const c, Dav1dPicture *const out) +{ + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + + const int drain = c->drain; + c->drain = 1; + + int res = gen_picture(c); + if (res < 0) + return res; + if (output_picture_ready(c)) return output_image(c, out, &c->out); if (c->n_fc > 1 && drain) return drain_picture(c, out); @@ -509,12 +541,12 @@ free(f->tile); free(f->lf.mask); free(f->lf.lr_mask); free(f->lf.level); free(f->lf.tx_lpf_right_edge[0]); - if (f->libaom_cm) dav1d_free_ref_mv_common(f->libaom_cm); - dav1d_free_aligned(f->lf.cdef_line[0][0][0]); + dav1d_refmvs_clear(&f->rf); + dav1d_free_aligned(f->lf.cdef_line_buf); dav1d_free_aligned(f->lf.lr_lpf_line[0]); } dav1d_free_aligned(c->fc); dav1d_data_unref_internal(&c->in); if (c->n_fc > 1 && c->frame_thread.out_delayed) { diff --git a/third_party/dav1d/src/looprestoration.h b/third_party/dav1d/src/looprestoration.h --- a/third_party/dav1d/src/looprestoration.h +++ b/third_party/dav1d/src/looprestoration.h @@ -70,11 +70,11 @@ typedef struct Dav1dLoopRestorationDSPContext { wienerfilter_fn wiener; selfguided_fn selfguided; } Dav1dLoopRestorationDSPContext; -bitfn_decls(void dav1d_loop_restoration_dsp_init, Dav1dLoopRestorationDSPContext *c); -bitfn_decls(void dav1d_loop_restoration_dsp_init_arm, Dav1dLoopRestorationDSPContext *c); +bitfn_decls(void dav1d_loop_restoration_dsp_init, Dav1dLoopRestorationDSPContext *c, int bpc); +bitfn_decls(void dav1d_loop_restoration_dsp_init_arm, Dav1dLoopRestorationDSPContext *c, int bpc); bitfn_decls(void dav1d_loop_restoration_dsp_init_x86, Dav1dLoopRestorationDSPContext *c); bitfn_decls(void dav1d_loop_restoration_dsp_init_ppc, Dav1dLoopRestorationDSPContext *c); #endif /* DAV1D_SRC_LOOPRESTORATION_H */ diff --git a/third_party/dav1d/src/looprestoration_tmpl.c b/third_party/dav1d/src/looprestoration_tmpl.c --- a/third_party/dav1d/src/looprestoration_tmpl.c +++ b/third_party/dav1d/src/looprestoration_tmpl.c @@ -571,17 +571,17 @@ p += PXSTRIDE(p_stride); } } } -COLD void bitfn(dav1d_loop_restoration_dsp_init)(Dav1dLoopRestorationDSPContext *const c) { +COLD void bitfn(dav1d_loop_restoration_dsp_init)(Dav1dLoopRestorationDSPContext *const c, int bpc) { c->wiener = wiener_c; c->selfguided = selfguided_c; #if HAVE_ASM #if ARCH_AARCH64 || ARCH_ARM - bitfn(dav1d_loop_restoration_dsp_init_arm)(c); + bitfn(dav1d_loop_restoration_dsp_init_arm)(c, bpc); #elif ARCH_PPC64LE bitfn(dav1d_loop_restoration_dsp_init_ppc)(c); #elif ARCH_X86 bitfn(dav1d_loop_restoration_dsp_init_x86)(c); #endif diff --git a/third_party/dav1d/src/lr_apply_tmpl.c b/third_party/dav1d/src/lr_apply_tmpl.c --- a/third_party/dav1d/src/lr_apply_tmpl.c +++ b/third_party/dav1d/src/lr_apply_tmpl.c @@ -71,15 +71,15 @@ } dst += 4 * PXSTRIDE(dst_stride); src += (stripe_h - 2) * PXSTRIDE(src_stride); - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { while (row + stripe_h <= row_h) { const int n_lines = 4 - (row + stripe_h + 1 == h); f->dsp->mc.resize(dst, dst_stride, src, src_stride, - dst_w, src_w, n_lines, f->resize_step[ss_hor], + dst_w, n_lines, src_w, f->resize_step[ss_hor], f->resize_start[ss_hor] HIGHBD_CALL_SUFFIX); row += stripe_h; // unmodified stripe_h for the 1st stripe stripe_h = 64 >> ss_ver; src += stripe_h * PXSTRIDE(src_stride); dst += n_lines * PXSTRIDE(dst_stride); diff --git a/third_party/dav1d/src/mc.h b/third_party/dav1d/src/mc.h --- a/third_party/dav1d/src/mc.h +++ b/third_party/dav1d/src/mc.h @@ -108,11 +108,11 @@ typedef decl_emu_edge_fn(*emu_edge_fn); #define decl_resize_fn(name) \ void (name)(pixel *dst, ptrdiff_t dst_stride, \ const pixel *src, ptrdiff_t src_stride, \ - int dst_w, int src_w, int h, int dx, int mx HIGHBD_DECL_SUFFIX) + int dst_w, int h, int src_w, int dx, int mx HIGHBD_DECL_SUFFIX) typedef decl_resize_fn(*resize_fn); typedef struct Dav1dMCDSPContext { mc_fn mc[N_2D_FILTERS]; mc_scaled_fn mc_scaled[N_2D_FILTERS]; diff --git a/third_party/dav1d/src/mc_tmpl.c b/third_party/dav1d/src/mc_tmpl.c --- a/third_party/dav1d/src/mc_tmpl.c +++ b/third_party/dav1d/src/mc_tmpl.c @@ -883,25 +883,25 @@ } } static void resize_c(pixel *dst, const ptrdiff_t dst_stride, const pixel *src, const ptrdiff_t src_stride, - const int dst_w, const int src_w, int h, + const int dst_w, int h, const int src_w, const int dx, const int mx0 HIGHBD_DECL_SUFFIX) { do { int mx = mx0, src_x = -1; for (int x = 0; x < dst_w; x++) { - const int16_t *const F = dav1d_resize_filter[mx >> 8]; - dst[x] = iclip_pixel((F[0] * src[iclip(src_x - 3, 0, src_w - 1)] + - F[1] * src[iclip(src_x - 2, 0, src_w - 1)] + - F[2] * src[iclip(src_x - 1, 0, src_w - 1)] + - F[3] * src[iclip(src_x + 0, 0, src_w - 1)] + - F[4] * src[iclip(src_x + 1, 0, src_w - 1)] + - F[5] * src[iclip(src_x + 2, 0, src_w - 1)] + - F[6] * src[iclip(src_x + 3, 0, src_w - 1)] + - F[7] * src[iclip(src_x + 4, 0, src_w - 1)] + + const int8_t *const F = dav1d_resize_filter[mx >> 8]; + dst[x] = iclip_pixel((-(F[0] * src[iclip(src_x - 3, 0, src_w - 1)] + + F[1] * src[iclip(src_x - 2, 0, src_w - 1)] + + F[2] * src[iclip(src_x - 1, 0, src_w - 1)] + + F[3] * src[iclip(src_x + 0, 0, src_w - 1)] + + F[4] * src[iclip(src_x + 1, 0, src_w - 1)] + + F[5] * src[iclip(src_x + 2, 0, src_w - 1)] + + F[6] * src[iclip(src_x + 3, 0, src_w - 1)] + + F[7] * src[iclip(src_x + 4, 0, src_w - 1)]) + 64) >> 7); mx += dx; src_x += mx >> 14; mx &= 0x3fff; } diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build --- a/third_party/dav1d/src/meson.build +++ b/third_party/dav1d/src/meson.build @@ -33,18 +33,19 @@ 'data.c', 'decode.c', 'dequant_tables.c', 'getbits.c', 'intra_edge.c', + 'itx_1d.c', 'lf_mask.c', 'log.c', 'msac.c', 'obu.c', 'picture.c', 'qm.c', 'ref.c', - 'ref_mvs.c', + 'refmvs.c', 'scan.c', 'tables.c', 'warpmv.c', 'wedge.c', ) @@ -99,33 +100,65 @@ 'arm/looprestoration_init_tmpl.c', 'arm/mc_init_tmpl.c', ) if host_machine.cpu_family() == 'aarch64' libdav1d_sources += files( - 'arm/64/cdef.S', - 'arm/64/ipred.S', - 'arm/64/itx.S', - 'arm/64/loopfilter.S', - 'arm/64/looprestoration.S', - 'arm/64/mc.S', + 'arm/64/looprestoration_common.S', 'arm/64/msac.S', ) + + if dav1d_bitdepths.contains('8') + libdav1d_sources += files( + 'arm/64/cdef.S', + 'arm/64/ipred.S', + 'arm/64/itx.S', + 'arm/64/loopfilter.S', + 'arm/64/looprestoration.S', + 'arm/64/mc.S', + ) + endif + + if dav1d_bitdepths.contains('16') + libdav1d_sources += files( + 'arm/64/cdef16.S', + 'arm/64/ipred16.S', + 'arm/64/loopfilter16.S', + 'arm/64/looprestoration16.S', + 'arm/64/mc16.S', + ) + endif elif host_machine.cpu_family().startswith('arm') libdav1d_sources += files( - 'arm/32/cdef.S', - 'arm/32/ipred.S', - 'arm/32/loopfilter.S', - 'arm/32/looprestoration.S', - 'arm/32/mc.S', ) + + if dav1d_bitdepths.contains('8') + libdav1d_sources += files( + 'arm/32/cdef.S', + 'arm/32/ipred.S', + 'arm/32/loopfilter.S', + 'arm/32/looprestoration.S', + 'arm/32/mc.S', + ) + endif + + if dav1d_bitdepths.contains('16') + libdav1d_sources += files( + ) + endif endif elif host_machine.cpu_family().startswith('x86') libdav1d_sources += files( 'x86/cpu.c', ) + if host_machine.cpu_family() == 'x86_64' + libdav1d_sources += files( + 'x86/msac_init.c', + ) + endif + libdav1d_tmpl_sources += files( 'x86/cdef_init_tmpl.c', 'x86/film_grain_init_tmpl.c', 'x86/ipred_init_tmpl.c', 'x86/itx_init_tmpl.c', @@ -140,18 +173,20 @@ 'x86/msac.asm', ) if dav1d_bitdepths.contains('8') libdav1d_sources_asm += files( - 'x86/cdef.asm', + 'x86/cdef_avx512.asm', + 'x86/cdef_avx2.asm', 'x86/film_grain.asm', 'x86/ipred.asm', 'x86/itx.asm', 'x86/loopfilter.asm', 'x86/looprestoration.asm', 'x86/mc.asm', 'x86/cdef_sse.asm', + 'x86/film_grain_ssse3.asm', 'x86/ipred_ssse3.asm', 'x86/itx_ssse3.asm', 'x86/loopfilter_ssse3.asm', 'x86/looprestoration_ssse3.asm', 'x86/mc_ssse3.asm', @@ -276,17 +311,22 @@ include_directories : dav1d_inc_dirs, dependencies : [ stdatomic_dependency, thread_dependency, thread_compat_dep, + libdl_dependency, ], c_args : [stackalign_flag, api_export_flags], version : dav1d_soname_version, soversion : dav1d_soversion, install : true, ) +dav1d_dep = declare_dependency(link_with: libdav1d, + include_directories : include_directories('../include/dav1d') +) + # # Generate pkg-config .pc file # pkg_mod = import('pkgconfig') pkg_mod.generate(libraries: libdav1d, diff --git a/third_party/dav1d/src/msac.h b/third_party/dav1d/src/msac.h --- a/third_party/dav1d/src/msac.h +++ b/third_party/dav1d/src/msac.h @@ -29,21 +29,25 @@ #define DAV1D_SRC_MSAC_H #include #include -#include "common/attributes.h" +#include "common/intops.h" typedef size_t ec_win; typedef struct MsacContext { const uint8_t *buf_pos; const uint8_t *buf_end; ec_win dif; unsigned rng; int cnt; int allow_update_cdf; + +#if ARCH_X86_64 && HAVE_ASM + unsigned (*symbol_adapt16)(struct MsacContext *s, uint16_t *cdf, size_t n_symbols); +#endif } MsacContext; #if HAVE_ASM #if ARCH_AARCH64 || ARCH_ARM #include "src/arm/msac.h" diff --git a/third_party/dav1d/src/msac.c b/third_party/dav1d/src/msac.c --- a/third_party/dav1d/src/msac.c +++ b/third_party/dav1d/src/msac.c @@ -36,11 +36,11 @@ #define EC_PROB_SHIFT 6 #define EC_MIN_PROB 4 // must be <= (1<buf_pos; const uint8_t *buf_end = s->buf_end; int c = EC_WIN_SIZE - s->cnt - 24; ec_win dif = s->dif; while (c >= 0 && buf_pos < buf_end) { @@ -55,47 +55,49 @@ /* Takes updated dif and range values, renormalizes them so that * 32768 <= rng < 65536 (reading more bytes from the stream into dif if * necessary), and stores them back in the decoder context. * dif: The new value of dif. * rng: The new value of the range. */ -static inline void ctx_norm(MsacContext *s, ec_win dif, unsigned rng) { +static inline void ctx_norm(MsacContext *const s, const ec_win dif, + const unsigned rng) +{ const int d = 15 ^ (31 ^ clz(rng)); assert(rng <= 65535U); s->cnt -= d; s->dif = ((dif + 1) << d) - 1; /* Shift in 1s in the LSBs */ s->rng = rng << d; if (s->cnt < 0) ctx_refill(s); } unsigned dav1d_msac_decode_bool_equi_c(MsacContext *const s) { - ec_win vw, dif = s->dif; - unsigned ret, v, r = s->rng; + const unsigned r = s->rng; + ec_win dif = s->dif; assert((dif >> (EC_WIN_SIZE - 16)) < r); // When the probability is 1/2, f = 16384 >> EC_PROB_SHIFT = 256 and we can // replace the multiply with a simple shift. - v = ((r >> 8) << 7) + EC_MIN_PROB; - vw = (ec_win)v << (EC_WIN_SIZE - 16); - ret = dif >= vw; - dif -= ret*vw; - v += ret*(r - 2*v); + unsigned v = ((r >> 8) << 7) + EC_MIN_PROB; + const ec_win vw = (ec_win)v << (EC_WIN_SIZE - 16); + const unsigned ret = dif >= vw; + dif -= ret * vw; + v += ret * (r - 2 * v); ctx_norm(s, dif, v); return !ret; } /* Decode a single binary value. * f: The probability that the bit is one * Return: The value decoded (0 or 1). */ unsigned dav1d_msac_decode_bool_c(MsacContext *const s, const unsigned f) { - ec_win vw, dif = s->dif; - unsigned ret, v, r = s->rng; + const unsigned r = s->rng; + ec_win dif = s->dif; assert((dif >> (EC_WIN_SIZE - 16)) < r); - v = ((r >> 8) * (f >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + EC_MIN_PROB; - vw = (ec_win)v << (EC_WIN_SIZE - 16); - ret = dif >= vw; - dif -= ret*vw; - v += ret*(r - 2*v); + unsigned v = ((r >> 8) * (f >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + EC_MIN_PROB; + const ec_win vw = (ec_win)v << (EC_WIN_SIZE - 16); + const unsigned ret = dif >= vw; + dif -= ret * vw; + v += ret * (r - 2 * v); ctx_norm(s, dif, v); return !ret; } int dav1d_msac_decode_subexp(MsacContext *const s, const int ref, @@ -194,7 +196,14 @@ s->buf_end = data + sz; s->dif = ((ec_win)1 << (EC_WIN_SIZE - 1)) - 1; s->rng = 0x8000; s->cnt = -15; s->allow_update_cdf = !disable_cdf_update_flag; + +#if ARCH_X86_64 && HAVE_ASM + s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt_c; + + dav1d_msac_init_x86(s); +#endif + ctx_refill(s); } diff --git a/third_party/dav1d/src/obu.c b/third_party/dav1d/src/obu.c --- a/third_party/dav1d/src/obu.c +++ b/third_party/dav1d/src/obu.c @@ -83,11 +83,11 @@ if (hdr->timing_info_present) { hdr->num_units_in_tick = dav1d_get_bits(gb, 32); hdr->time_scale = dav1d_get_bits(gb, 32); hdr->equal_picture_interval = dav1d_get_bits(gb, 1); if (hdr->equal_picture_interval) { - unsigned num_ticks_per_picture = dav1d_get_vlc(gb); + const unsigned num_ticks_per_picture = dav1d_get_vlc(gb); if (num_ticks_per_picture == 0xFFFFFFFFU) goto error; hdr->num_ticks_per_picture = num_ticks_per_picture + 1; } @@ -109,19 +109,19 @@ hdr->display_model_info_present = dav1d_get_bits(gb, 1); hdr->num_operating_points = dav1d_get_bits(gb, 5) + 1; for (int i = 0; i < hdr->num_operating_points; i++) { struct Dav1dSequenceHeaderOperatingPoint *const op = &hdr->operating_points[i]; - struct Dav1dSequenceHeaderOperatingParameterInfo *const opi = - &hdr->operating_parameter_info[i]; op->idc = dav1d_get_bits(gb, 12); op->major_level = 2 + dav1d_get_bits(gb, 3); op->minor_level = dav1d_get_bits(gb, 2); op->tier = op->major_level > 3 ? dav1d_get_bits(gb, 1) : 0; op->decoder_model_param_present = hdr->decoder_model_info_present && dav1d_get_bits(gb, 1); if (op->decoder_model_param_present) { + struct Dav1dSequenceHeaderOperatingParameterInfo *const opi = + &hdr->operating_parameter_info[i]; opi->decoder_buffer_delay = dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); opi->encoder_buffer_delay = dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); opi->low_delay_mode = dav1d_get_bits(gb, 1); @@ -130,14 +130,13 @@ hdr->display_model_info_present && dav1d_get_bits(gb, 1); if (op->display_model_param_present) { op->initial_display_delay = dav1d_get_bits(gb, 4) + 1; } } - if (c->operating_point < hdr->num_operating_points) - c->operating_point_idc = hdr->operating_points[c->operating_point].idc; - else - c->operating_point_idc = hdr->operating_points[0].idc; + const int op_idx = + c->operating_point < hdr->num_operating_points ? c->operating_point : 0; + c->operating_point_idc = hdr->operating_points[op_idx].idc; #if DEBUG_SEQ_HDR printf("SEQHDR: post-operating-points: off=%ld\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif } @@ -293,11 +292,11 @@ Dav1dFrameHeader *const hdr = c->frame_hdr; if (use_ref) { for (int i = 0; i < 7; i++) { if (dav1d_get_bits(gb, 1)) { - Dav1dThreadPicture *const ref = + const Dav1dThreadPicture *const ref = &c->refs[c->frame_hdr->refidx[i]].p; if (!ref->p.data[0]) return -1; hdr->width[1] = ref->p.p.w; hdr->height = ref->p.p.h; hdr->render_width = ref->p.frame_hdr->render_width; @@ -341,11 +340,11 @@ hdr->render_height = hdr->height; } return 0; } -static inline int tile_log2(int sz, int tgt) { +static inline int tile_log2(const int sz, const int tgt) { int k; for (k = 0; (sz << k) < tgt; k++) ; return k; } @@ -360,11 +359,10 @@ #if DEBUG_FRAME_HDR const uint8_t *const init_ptr = gb->ptr; #endif const Dav1dSequenceHeader *const seqhdr = c->seq_hdr; Dav1dFrameHeader *const hdr = c->frame_hdr; - int res; hdr->show_existing_frame = !seqhdr->reduced_still_picture_header && dav1d_get_bits(gb, 1); #if DEBUG_FRAME_HDR printf("HDR: post-show_existing_frame: off=%ld\n", @@ -442,11 +440,11 @@ hdr->refresh_frame_flags = (hdr->frame_type == DAV1D_FRAME_TYPE_KEY && hdr->show_frame) ? 0xff : dav1d_get_bits(gb, 8); if (hdr->refresh_frame_flags != 0xff && hdr->error_resilient_mode && seqhdr->order_hint) for (int i = 0; i < 8; i++) dav1d_get_bits(gb, seqhdr->order_hint_n_bits); - if ((res = read_frame_size(c, gb, 0)) < 0) goto error; + if (read_frame_size(c, gb, 0) < 0) goto error; hdr->allow_intrabc = hdr->allow_screen_content_tools && !hdr->super_res.enabled && dav1d_get_bits(gb, 1); hdr->use_ref_frame_mvs = 0; } else { hdr->allow_intrabc = 0; @@ -477,11 +475,11 @@ used_frame[hdr->refidx[0]] = 1; used_frame[hdr->refidx[3]] = 1; int latest_frame_offset = -1; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (!used_frame[i] && hint >= current_frame_offset && hint >= latest_frame_offset) { hdr->refidx[6] = i; latest_frame_offset = hint; @@ -490,11 +488,11 @@ if (latest_frame_offset != -1) used_frame[hdr->refidx[6]] = 1; int earliest_frame_offset = INT_MAX; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (!used_frame[i] && hint >= current_frame_offset && hint < earliest_frame_offset) { hdr->refidx[4] = i; earliest_frame_offset = hint; @@ -503,11 +501,11 @@ if (earliest_frame_offset != INT_MAX) used_frame[hdr->refidx[4]] = 1; earliest_frame_offset = INT_MAX; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (!used_frame[i] && hint >= current_frame_offset && (hint < earliest_frame_offset)) { hdr->refidx[5] = i; earliest_frame_offset = hint; @@ -518,11 +516,11 @@ for (int i = 1; i < 7; i++) { if (hdr->refidx[i] < 0) { latest_frame_offset = -1; for (int j = 0; j < 8; j++) { - int hint = shifted_frame_offset[j]; + const int hint = shifted_frame_offset[j]; if (!used_frame[j] && hint < current_frame_offset && hint >= latest_frame_offset) { hdr->refidx[i] = j; latest_frame_offset = hint; @@ -534,11 +532,11 @@ } earliest_frame_offset = INT_MAX; int ref = -1; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (hint < earliest_frame_offset) { ref = i; earliest_frame_offset = hint; } } @@ -553,11 +551,11 @@ if (seqhdr->frame_id_numbers_present) dav1d_get_bits(gb, seqhdr->delta_frame_id_n_bits); } const int use_ref = !hdr->error_resilient_mode && hdr->frame_size_override; - if ((res = read_frame_size(c, gb, use_ref)) < 0) goto error; + if (read_frame_size(c, gb, use_ref) < 0) goto error; hdr->hp = !hdr->force_integer_mv && dav1d_get_bits(gb, 1); hdr->subpel_filter_mode = dav1d_get_bits(gb, 1) ? DAV1D_FILTER_SWITCHABLE : dav1d_get_bits(gb, 2); hdr->switchable_motion_mode = dav1d_get_bits(gb, 1); hdr->use_ref_frame_mvs = !hdr->error_resilient_mode && @@ -577,19 +575,19 @@ #endif // tile data hdr->tiling.uniform = dav1d_get_bits(gb, 1); const int sbsz_min1 = (64 << seqhdr->sb128) - 1; - int sbsz_log2 = 6 + seqhdr->sb128; - int sbw = (hdr->width[0] + sbsz_min1) >> sbsz_log2; - int sbh = (hdr->height + sbsz_min1) >> sbsz_log2; - int max_tile_width_sb = 4096 >> sbsz_log2; - int max_tile_area_sb = 4096 * 2304 >> (2 * sbsz_log2); + const int sbsz_log2 = 6 + seqhdr->sb128; + const int sbw = (hdr->width[0] + sbsz_min1) >> sbsz_log2; + const int sbh = (hdr->height + sbsz_min1) >> sbsz_log2; + const int max_tile_width_sb = 4096 >> sbsz_log2; + const int max_tile_area_sb = 4096 * 2304 >> (2 * sbsz_log2); hdr->tiling.min_log2_cols = tile_log2(max_tile_width_sb, sbw); hdr->tiling.max_log2_cols = tile_log2(1, imin(sbw, DAV1D_MAX_TILE_COLS)); hdr->tiling.max_log2_rows = tile_log2(1, imin(sbh, DAV1D_MAX_TILE_ROWS)); - int min_log2_tiles = imax(tile_log2(max_tile_area_sb, sbw * sbh), + const int min_log2_tiles = imax(tile_log2(max_tile_area_sb, sbw * sbh), hdr->tiling.min_log2_cols); if (hdr->tiling.uniform) { for (hdr->tiling.log2_cols = hdr->tiling.min_log2_cols; hdr->tiling.log2_cols < hdr->tiling.max_log2_cols && dav1d_get_bits(gb, 1); hdr->tiling.log2_cols++) ; @@ -619,11 +617,11 @@ sbx += tile_w; widest_tile = imax(widest_tile, tile_w); } hdr->tiling.log2_cols = tile_log2(1, hdr->tiling.cols); if (min_log2_tiles) max_tile_area_sb >>= min_log2_tiles + 1; - int max_tile_height_sb = imax(max_tile_area_sb / widest_tile, 1); + const int max_tile_height_sb = imax(max_tile_area_sb / widest_tile, 1); hdr->tiling.rows = 0; for (int sby = 0; sby < sbh && hdr->tiling.rows < DAV1D_MAX_TILE_ROWS; hdr->tiling.rows++) { const int tile_height_sb = imin(sbh - sby, max_tile_height_sb); const int tile_h = (tile_height_sb > 1) ? @@ -655,11 +653,11 @@ hdr->quant.ydc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; if (!seqhdr->monochrome) { // If the sequence header says that delta_q might be different // for U, V, we must check whether it actually is for this // frame. - int diff_uv_delta = seqhdr->separate_uv_delta_q ? dav1d_get_bits(gb, 1) : 0; + const int diff_uv_delta = seqhdr->separate_uv_delta_q ? dav1d_get_bits(gb, 1) : 0; hdr->quant.udc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; hdr->quant.uac_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; if (diff_uv_delta) { hdr->quant.vdc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; hdr->quant.vac_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; @@ -915,14 +913,13 @@ (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif hdr->skip_mode_allowed = 0; if (hdr->switchable_comp_refs && hdr->frame_type & 1 && seqhdr->order_hint) { const unsigned poc = hdr->frame_offset; - unsigned off_before[2] = { 0xFFFFFFFF, 0xFFFFFFFF }; + unsigned off_before = 0xFFFFFFFFU; int off_after = -1; - int off_before_idx[2], off_after_idx; - off_before_idx[0] = 0; + int off_before_idx, off_after_idx; for (int i = 0; i < 7; i++) { if (!c->refs[hdr->refidx[i]].p.p.data[0]) return DAV1D_ERR(EINVAL); const unsigned refpoc = c->refs[hdr->refidx[i]].p.p.frame_hdr->frame_offset; const int diff = get_poc_diff(seqhdr->order_hint_n_bits, refpoc, poc); @@ -931,40 +928,46 @@ off_after, refpoc) > 0) { off_after = refpoc; off_after_idx = i; } - } else if (diff < 0) { - if (off_before[0] == 0xFFFFFFFFU || - get_poc_diff(seqhdr->order_hint_n_bits, - refpoc, off_before[0]) > 0) - { - off_before[1] = off_before[0]; - off_before[0] = refpoc; - off_before_idx[1] = off_before_idx[0]; - off_before_idx[0] = i; - } else if (refpoc != off_before[0] && - (off_before[1] == 0xFFFFFFFFU || - get_poc_diff(seqhdr->order_hint_n_bits, - refpoc, off_before[1]) > 0)) - { - off_before[1] = refpoc; - off_before_idx[1] = i; - } + } else if (diff < 0 && (off_before == 0xFFFFFFFFU || + get_poc_diff(seqhdr->order_hint_n_bits, + refpoc, off_before) > 0)) + { + off_before = refpoc; + off_before_idx = i; } } - if (off_before[0] != 0xFFFFFFFFU && off_after != -1) { - hdr->skip_mode_refs[0] = imin(off_before_idx[0], off_after_idx); - hdr->skip_mode_refs[1] = imax(off_before_idx[0], off_after_idx); + if (off_before != 0xFFFFFFFFU && off_after != -1) { + hdr->skip_mode_refs[0] = imin(off_before_idx, off_after_idx); + hdr->skip_mode_refs[1] = imax(off_before_idx, off_after_idx); hdr->skip_mode_allowed = 1; - } else if (off_before[0] != 0xFFFFFFFFU && - off_before[1] != 0xFFFFFFFFU) - { - hdr->skip_mode_refs[0] = imin(off_before_idx[0], off_before_idx[1]); - hdr->skip_mode_refs[1] = imax(off_before_idx[0], off_before_idx[1]); - hdr->skip_mode_allowed = 1; + } else if (off_before != 0xFFFFFFFFU) { + unsigned off_before2 = 0xFFFFFFFFU; + int off_before2_idx; + for (int i = 0; i < 7; i++) { + if (!c->refs[hdr->refidx[i]].p.p.data[0]) return DAV1D_ERR(EINVAL); + const unsigned refpoc = c->refs[hdr->refidx[i]].p.p.frame_hdr->frame_offset; + if (get_poc_diff(seqhdr->order_hint_n_bits, + refpoc, off_before) < 0) { + if (off_before2 == 0xFFFFFFFFU || + get_poc_diff(seqhdr->order_hint_n_bits, + refpoc, off_before2) > 0) + { + off_before2 = refpoc; + off_before2_idx = i; + } + } + } + + if (off_before2 != 0xFFFFFFFFU) { + hdr->skip_mode_refs[0] = imin(off_before_idx, off_before2_idx); + hdr->skip_mode_refs[1] = imax(off_before_idx, off_before2_idx); + hdr->skip_mode_allowed = 1; + } } } hdr->skip_mode_enabled = hdr->skip_mode_allowed ? dav1d_get_bits(gb, 1) : 0; #if DEBUG_FRAME_HDR printf("HDR: post-extskip: off=%ld\n", @@ -1046,11 +1049,11 @@ const int refidx = dav1d_get_bits(gb, 3); int i; for (i = 0; i < 7; i++) if (hdr->refidx[i] == refidx) break; - if (i == 7 || !c->refs[refidx].p.p.frame_hdr) goto error; + if (i == 7 || !c->refs[refidx].p.p.frame_hdr) goto error; hdr->film_grain.data = c->refs[refidx].p.p.frame_hdr->film_grain.data; hdr->film_grain.data.seed = seed; } else { Dav1dFilmGrainData *const fgd = &hdr->film_grain.data; fgd->seed = seed; @@ -1126,14 +1129,12 @@ dav1d_log(c, "Error parsing frame header\n"); return DAV1D_ERR(EINVAL); } static void parse_tile_hdr(Dav1dContext *const c, GetBits *const gb) { - int have_tile_pos = 0; const int n_tiles = c->frame_hdr->tiling.cols * c->frame_hdr->tiling.rows; - if (n_tiles > 1) - have_tile_pos = dav1d_get_bits(gb, 1); + const int have_tile_pos = n_tiles > 1 ? dav1d_get_bits(gb, 1) : 0; if (have_tile_pos) { const int n_bits = c->frame_hdr->tiling.log2_cols + c->frame_hdr->tiling.log2_rows; c->tile[c->n_tile_data].start = dav1d_get_bits(gb, n_bits); @@ -1144,21 +1145,21 @@ } } // Check that we haven't read more than obu_len bytes from the buffer // since init_bit_pos. -static int -check_for_overrun(Dav1dContext *const c, GetBits *const gb, - unsigned init_bit_pos, unsigned obu_len) +static int check_for_overrun(Dav1dContext *const c, GetBits *const gb, + const unsigned init_bit_pos, + const unsigned obu_len) { // Make sure we haven't actually read past the end of the gb buffer if (gb->error) { dav1d_log(c, "Overrun in OBU bit buffer\n"); return 1; } - unsigned pos = dav1d_get_bits_pos(gb); + const unsigned pos = dav1d_get_bits_pos(gb); // We assume that init_bit_pos was the bit position of the buffer // at some point in the past, so cannot be smaller than pos. assert (init_bit_pos <= pos); @@ -1168,11 +1169,11 @@ } return 0; } -int dav1d_parse_obus(Dav1dContext *const c, Dav1dData *const in, int global) { +int dav1d_parse_obus(Dav1dContext *const c, Dav1dData *const in, const int global) { GetBits gb; int res; dav1d_init_get_bits(&gb, in->data, in->sz); @@ -1189,15 +1190,12 @@ spatial_id = dav1d_get_bits(&gb, 2); dav1d_get_bits(&gb, 3); // reserved } // obu length field - unsigned len = 0; - if (has_length_field) - len = dav1d_get_uleb128(&gb); - else - len = (int) in->sz - 1 - has_extension; + const unsigned len = has_length_field ? + dav1d_get_uleb128(&gb) : (unsigned) in->sz - 1 - has_extension; if (gb.error) goto error; const unsigned init_bit_pos = dav1d_get_bits_pos(&gb); const unsigned init_byte_pos = init_bit_pos >> 3; const unsigned pkt_bytelen = init_byte_pos + len; @@ -1435,11 +1433,11 @@ // Don't take into account meta_type bytes payload_size -= meta_type_len; int country_code_extension_byte = 0; - int country_code = dav1d_get_bits(&gb, 8); + const int country_code = dav1d_get_bits(&gb, 8); payload_size--; if (country_code == 0xFF) { country_code_extension_byte = dav1d_get_bits(&gb, 8); payload_size--; } diff --git a/third_party/dav1d/src/picture.c b/third_party/dav1d/src/picture.c --- a/third_party/dav1d/src/picture.c +++ b/third_party/dav1d/src/picture.c @@ -66,11 +66,11 @@ p->stride[0] = y_stride; p->stride[1] = uv_stride; const size_t y_sz = y_stride * aligned_h; const size_t uv_sz = uv_stride * (aligned_h >> ss_ver); const size_t pic_size = y_sz + 2 * uv_sz + DAV1D_PICTURE_ALIGNMENT; - uint8_t *data = dav1d_alloc_aligned(pic_size, DAV1D_PICTURE_ALIGNMENT); + uint8_t *const data = dav1d_alloc_aligned(pic_size, DAV1D_PICTURE_ALIGNMENT); if (!data) return DAV1D_ERR(ENOMEM); p->data[0] = data; p->data[1] = has_chroma ? data + y_sz : NULL; p->data[2] = has_chroma ? data + y_sz + uv_sz : NULL; @@ -102,31 +102,32 @@ pic_ctx->allocator.release_picture_callback(&pic_ctx->pic, pic_ctx->allocator.cookie); free(pic_ctx); } -static int picture_alloc_with_edges(Dav1dContext *const c, Dav1dPicture *const p, +static int picture_alloc_with_edges(Dav1dContext *const c, + Dav1dPicture *const p, const int w, const int h, - Dav1dSequenceHeader *seq_hdr, Dav1dRef *seq_hdr_ref, - Dav1dFrameHeader *frame_hdr, Dav1dRef *frame_hdr_ref, - Dav1dContentLightLevel *content_light, Dav1dRef *content_light_ref, - Dav1dMasteringDisplay *mastering_display, Dav1dRef *mastering_display_ref, - Dav1dITUTT35 *itut_t35, Dav1dRef *itut_t35_ref, - const int bpc, const Dav1dDataProps *props, + Dav1dSequenceHeader *const seq_hdr, Dav1dRef *const seq_hdr_ref, + Dav1dFrameHeader *const frame_hdr, Dav1dRef *const frame_hdr_ref, + Dav1dContentLightLevel *const content_light, Dav1dRef *const content_light_ref, + Dav1dMasteringDisplay *const mastering_display, Dav1dRef *const mastering_display_ref, + Dav1dITUTT35 *const itut_t35, Dav1dRef *const itut_t35_ref, + const int bpc, + const Dav1dDataProps *const props, Dav1dPicAllocator *const p_allocator, const size_t extra, void **const extra_ptr) { if (p->data[0]) { dav1d_log(c, "Picture already allocated!\n"); return -1; } assert(bpc > 0 && bpc <= 16); struct pic_ctx_context *pic_ctx = malloc(extra + sizeof(struct pic_ctx_context)); - if (pic_ctx == NULL) { + if (pic_ctx == NULL) return DAV1D_ERR(ENOMEM); - } p->p.w = w; p->p.h = h; p->seq_hdr = seq_hdr; p->frame_hdr = frame_hdr; @@ -134,11 +135,11 @@ p->mastering_display = mastering_display; p->itut_t35 = itut_t35; p->p.layout = seq_hdr->layout; p->p.bpc = bpc; dav1d_data_props_set_defaults(&p->m); - int res = p_allocator->alloc_picture_callback(p, p_allocator->cookie); + const int res = p_allocator->alloc_picture_callback(p, p_allocator->cookie); if (res < 0) { free(pic_ctx); return res; } @@ -248,12 +249,12 @@ *dst = *src; memset(src, 0, sizeof(*src)); } -void dav1d_thread_picture_ref(Dav1dThreadPicture *dst, - const Dav1dThreadPicture *src) +void dav1d_thread_picture_ref(Dav1dThreadPicture *const dst, + const Dav1dThreadPicture *const src) { dav1d_picture_ref(&dst->p, &src->p); dst->t = src->t; dst->visible = src->visible; dst->progress = src->progress; diff --git a/third_party/dav1d/src/ppc/cdef_init_tmpl.c b/third_party/dav1d/src/ppc/cdef_init_tmpl.c --- a/third_party/dav1d/src/ppc/cdef_init_tmpl.c +++ b/third_party/dav1d/src/ppc/cdef_init_tmpl.c @@ -51,11 +51,11 @@ return vec_sel(min, neg, mask); } static inline void copy4xN(uint16_t *tmp, const ptrdiff_t tmp_stride, const uint8_t *src, const ptrdiff_t src_stride, - const uint8_t (*left)[2], uint8_t *const top[2], + const uint8_t (*left)[2], const uint8_t *const top, const int w, const int h, const enum CdefEdgeFlags edges) { const u16x8 fill = vec_splats((uint16_t)INT16_MAX); @@ -68,12 +68,12 @@ if (!(edges & CDEF_HAVE_TOP)) { l0 = fill; l1 = fill; y_start = 0; } else { - l0 = u8h_to_u16(vec_vsx_ld(0, top[0] - 2)); - l1 = u8h_to_u16(vec_vsx_ld(0, top[1] - 2)); + l0 = u8h_to_u16(vec_vsx_ld(0, top + 0 * src_stride - 2)); + l1 = u8h_to_u16(vec_vsx_ld(0, top + 1 * src_stride - 2)); } vec_st(l0, 0, tmp - 2 * 8); vec_st(l1, 0, tmp - 1 * 8); @@ -113,11 +113,11 @@ } } static inline void copy8xN(uint16_t *tmp, const ptrdiff_t tmp_stride, const uint8_t *src, const ptrdiff_t src_stride, - const uint8_t (*left)[2], uint8_t *const top[2], + const uint8_t (*left)[2], const uint8_t *const top, const int w, const int h, const enum CdefEdgeFlags edges) { const u16x8 fill = vec_splats((uint16_t)INT16_MAX); @@ -132,12 +132,12 @@ l0l = fill; l1h = fill; l1l = fill; y_start = 0; } else { - u8x16 l0 = vec_vsx_ld(0, top[0] - 2); - u8x16 l1 = vec_vsx_ld(0, top[1] - 2); + u8x16 l0 = vec_vsx_ld(0, top + 0 * src_stride - 2); + u8x16 l1 = vec_vsx_ld(0, top + 1 * src_stride - 2); l0h = u8h_to_u16(l0); l0l = u8l_to_u16(l0); l1h = u8h_to_u16(l1); l1l = u8l_to_u16(l1); } @@ -273,11 +273,11 @@ sum = vec_add(sum, p ## sum0); \ sum = vec_add(sum, p ## sum1); static inline void filter_4xN(pixel *dst, const ptrdiff_t dst_stride, - const pixel (*left)[2], /*const*/ pixel *const top[2], + const pixel (*left)[2], const pixel *const top, const int w, const int h, const int pri_strength, const int sec_strength, const int dir, const int damping, const enum CdefEdgeFlags edges, const ptrdiff_t tmp_stride, uint16_t *tmp) { @@ -362,11 +362,11 @@ } } static inline void filter_8xN(pixel *dst, const ptrdiff_t dst_stride, - const pixel (*left)[2], /*const*/ pixel *const top[2], + const pixel (*left)[2], const pixel *const top, const int w, const int h, const int pri_strength, const int sec_strength, const int dir, const int damping, const enum CdefEdgeFlags edges, const ptrdiff_t tmp_stride, uint16_t *tmp) { @@ -454,11 +454,11 @@ #define cdef_fn(w, h, tmp_stride) \ static void cdef_filter_##w##x##h##_vsx(pixel *const dst, \ const ptrdiff_t dst_stride, \ const pixel (*left)[2], \ - /*const*/ pixel *const top[2], \ + const pixel *const top, \ const int pri_strength, \ const int sec_strength, \ const int dir, \ const int damping, \ const enum CdefEdgeFlags edges) \ diff --git a/third_party/dav1d/src/recon_tmpl.c b/third_party/dav1d/src/recon_tmpl.c --- a/third_party/dav1d/src/recon_tmpl.c +++ b/third_party/dav1d/src/recon_tmpl.c @@ -68,34 +68,34 @@ if (chroma) { const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; const int not_one_blk = b_dim[2] - (!!b_dim[2] && ss_hor) > t_dim->lw || b_dim[3] - (!!b_dim[3] && ss_ver) > t_dim->lh; - int ca, cl; + unsigned ca, cl; -#define MERGE_CTX(dir, type, mask) \ - c##dir = !!((*(const type *) dir) & mask); \ +#define MERGE_CTX(dir, type, no_val) \ + c##dir = *(const type *) dir != no_val; \ break switch (t_dim->lw) { /* For some reason the MSVC CRT _wassert() function is not flagged as * __declspec(noreturn), so when using those headers the compiler will * expect execution to continue after an assertion has been triggered * and will therefore complain about the use of uninitialized variables * when compiled in debug mode if we put the default case at the end. */ default: assert(0); /* fall-through */ - case TX_4X4: MERGE_CTX(a, uint8_t, 0x3F); - case TX_8X8: MERGE_CTX(a, uint16_t, 0x3F3F); - case TX_16X16: MERGE_CTX(a, uint32_t, 0x3F3F3F3FU); - case TX_32X32: MERGE_CTX(a, uint64_t, 0x3F3F3F3F3F3F3F3FULL); + case TX_4X4: MERGE_CTX(a, uint8_t, 0x40); + case TX_8X8: MERGE_CTX(a, uint16_t, 0x4040); + case TX_16X16: MERGE_CTX(a, uint32_t, 0x40404040U); + case TX_32X32: MERGE_CTX(a, uint64_t, 0x4040404040404040ULL); } switch (t_dim->lh) { default: assert(0); /* fall-through */ - case TX_4X4: MERGE_CTX(l, uint8_t, 0x3F); - case TX_8X8: MERGE_CTX(l, uint16_t, 0x3F3F); - case TX_16X16: MERGE_CTX(l, uint32_t, 0x3F3F3F3FU); - case TX_32X32: MERGE_CTX(l, uint64_t, 0x3F3F3F3F3F3F3F3FULL); + case TX_4X4: MERGE_CTX(l, uint8_t, 0x40); + case TX_8X8: MERGE_CTX(l, uint16_t, 0x4040); + case TX_16X16: MERGE_CTX(l, uint32_t, 0x40404040U); + case TX_32X32: MERGE_CTX(l, uint64_t, 0x4040404040404040ULL); } #undef MERGE_CTX return 7 + not_one_blk * 3 + ca + cl; } else if (b_dim[2] == t_dim->lw && b_dim[3] == t_dim->lh) { @@ -350,17 +350,21 @@ // transform type (chroma: derived, luma: explicitly coded) if (lossless) { assert(t_dim->max == TX_4X4); *txtp = WHT_WHT; - } else if (!f->frame_hdr->segmentation.qidx[b->seg_id] || - t_dim->max + intra >= TX_64X64) - { + } else if (t_dim->max + intra >= TX_64X64) { *txtp = DCT_DCT; } else if (chroma) { + // inferred from either the luma txtp (inter) or a LUT (intra) *txtp = intra ? dav1d_txtp_from_uvmode[b->uv_mode] : get_uv_inter_txtp(t_dim, *txtp); + } else if (!f->frame_hdr->segmentation.qidx[b->seg_id]) { + // In libaom, lossless is checked by a literal qidx == 0, but not all + // such blocks are actually lossless. The remainder gets an implicit + // transform type (for luma) + *txtp = DCT_DCT; } else { unsigned idx; if (intra) { const enum IntraPredMode y_mode_nofilt = b->y_mode == FILTER_PRED ? dav1d_filter_mode_to_y_mode[b->y_angle] : b->y_mode; @@ -768,14 +772,15 @@ const int cw4 = (w4 + ss_hor) >> ss_hor, ch4 = (h4 + ss_ver) >> ss_ver; assert(f->frame_thread.pass == 1); assert(!b->skip); const TxfmInfo *const uv_t_dim = &dav1d_txfm_dimensions[b->uvtx]; const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[b->intra ? b->tx : b->max_ytx]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; for (int init_y = 0; init_y < h4; init_y += 16) { + const int sub_h4 = imin(h4, 16 + init_y); for (int init_x = 0; init_x < w4; init_x += 16) { - const int sub_h4 = imin(h4, 16 + init_y); const int sub_w4 = imin(w4, init_x + 16); int y_off = !!init_y, y, x; for (y = init_y, t->by += init_y; y < sub_h4; y += t_dim->h, t->by += t_dim->h, y_off++) { @@ -784,11 +789,11 @@ int x_off = !!init_x; for (x = init_x, t->bx += init_x; x < sub_w4; x += t_dim->w, t->bx += t_dim->w, x_off++) { if (!b->intra) { - read_coef_tree(t, bs, b, b->max_ytx, 0, b->tx_split, + read_coef_tree(t, bs, b, b->max_ytx, 0, tx_split, x_off, y_off, NULL); } else { uint8_t cf_ctx = 0x40; enum TxfmType txtp; const int eob = cbi[t->bx].eob[0] = @@ -925,12 +930,12 @@ HIGHBD_CALL_SUFFIX); } } else { assert(refp != &f->sr_cur); - int orig_pos_y = (by * v_mul << 4) + mvy * (1 << !ss_ver); - int orig_pos_x = (bx * h_mul << 4) + mvx * (1 << !ss_hor); + const int orig_pos_y = (by * v_mul << 4) + mvy * (1 << !ss_ver); + const int orig_pos_x = (bx * h_mul << 4) + mvx * (1 << !ss_hor); #define scale_mv(res, val, scale) do { \ const int64_t tmp = (int64_t)(val) * scale + (scale - 0x4000) * 8; \ res = apply_sign64((int) ((llabs(tmp) + 128) >> 8), tmp) + 32; \ } while (0) int pos_y, pos_x; @@ -992,11 +997,11 @@ const uint8_t *const b_dim, const int pl, const int bx4, const int by4, const int w4, const int h4) { assert(!(t->bx & 1) && !(t->by & 1)); const Dav1dFrameContext *const f = t->f; - const refmvs *const r = &f->mvs[t->by * f->b4_stride + t->bx]; + /*const*/ refmvs_block **r = &t->rt.r[(t->by & 31) + 5]; pixel *const lap = bitfn(t->scratch.lap); const int ss_ver = !!pl && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = !!pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; int res; @@ -1004,20 +1009,19 @@ if (t->by > t->ts->tiling.row_start && (!pl || b_dim[0] * h_mul + b_dim[1] * v_mul >= 16)) { for (int i = 0, x = 0; x < w4 && i < imin(b_dim[2], 4); ) { // only odd blocks are considered for overlap handling, hence +1 - const refmvs *const a_r = &r[x - f->b4_stride + 1]; - const uint8_t *const a_b_dim = - dav1d_block_dimensions[dav1d_sbtype_to_bs[a_r->sb_type]]; + const refmvs_block *const a_r = &r[-1][t->bx + x + 1]; + const uint8_t *const a_b_dim = dav1d_block_dimensions[a_r->bs]; - if (a_r->ref[0] > 0) { + if (a_r->ref.ref[0] > 0) { const int ow4 = iclip(a_b_dim[0], 2, b_dim[0]); const int oh4 = imin(b_dim[1], 16) >> 1; res = mc(t, lap, NULL, ow4 * h_mul * sizeof(pixel), ow4, (oh4 * 3 + 3) >> 2, - t->bx + x, t->by, pl, a_r->mv[0], - &f->refp[a_r->ref[0] - 1], a_r->ref[0] - 1, + t->bx + x, t->by, pl, a_r->mv.mv[0], + &f->refp[a_r->ref.ref[0] - 1], a_r->ref.ref[0] - 1, dav1d_filter_2d[t->a->filter[1][bx4 + x + 1]][t->a->filter[0][bx4 + x + 1]]); if (res) return res; f->dsp->mc.blend_h(&dst[x * h_mul], dst_stride, lap, h_mul * ow4, v_mul * oh4); i++; @@ -1027,20 +1031,19 @@ } if (t->bx > t->ts->tiling.col_start) for (int i = 0, y = 0; y < h4 && i < imin(b_dim[3], 4); ) { // only odd blocks are considered for overlap handling, hence +1 - const refmvs *const l_r = &r[(y + 1) * f->b4_stride - 1]; - const uint8_t *const l_b_dim = - dav1d_block_dimensions[dav1d_sbtype_to_bs[l_r->sb_type]]; + const refmvs_block *const l_r = &r[y + 1][t->bx - 1]; + const uint8_t *const l_b_dim = dav1d_block_dimensions[l_r->bs]; - if (l_r->ref[0] > 0) { + if (l_r->ref.ref[0] > 0) { const int ow4 = imin(b_dim[0], 16) >> 1; const int oh4 = iclip(l_b_dim[1], 2, b_dim[1]); res = mc(t, lap, NULL, h_mul * ow4 * sizeof(pixel), ow4, oh4, - t->bx, t->by + y, pl, l_r->mv[0], - &f->refp[l_r->ref[0] - 1], l_r->ref[0] - 1, + t->bx, t->by + y, pl, l_r->mv.mv[0], + &f->refp[l_r->ref.ref[0] - 1], l_r->ref.ref[0] - 1, dav1d_filter_2d[t->l.filter[1][by4 + y + 1]][t->l.filter[0][by4 + y + 1]]); if (res) return res; f->dsp->mc.blend_v(&dst[y * v_mul * PXSTRIDE(dst_stride)], dst_stride, lap, h_mul * ow4, v_mul * oh4); i++; @@ -1142,10 +1145,12 @@ const int cbw4 = (bw4 + ss_hor) >> ss_hor, cbh4 = (bh4 + ss_ver) >> ss_ver; const int intra_edge_filter_flag = f->seq_hdr->intra_edge_filter << 10; for (int init_y = 0; init_y < h4; init_y += 16) { + const int sub_h4 = imin(h4, 16 + init_y); + const int sub_ch4 = imin(ch4, (init_y + 16) >> ss_ver); for (int init_x = 0; init_x < w4; init_x += 16) { if (b->pal_sz[0]) { pixel *dst = ((pixel *) f->cur.data[0]) + 4 * (t->by * PXSTRIDE(f->cur.stride[0]) + t->bx); const uint8_t *pal_idx; @@ -1172,11 +1177,10 @@ const int sb_has_tr = init_x + 16 < w4 ? 1 : init_y ? 0 : intra_edge_flags & EDGE_I444_TOP_HAS_RIGHT; const int sb_has_bl = init_x ? 0 : init_y + 16 < h4 ? 1 : intra_edge_flags & EDGE_I444_LEFT_HAS_BOTTOM; int y, x; - const int sub_h4 = imin(h4, 16 + init_y); const int sub_w4 = imin(w4, init_x + 16); for (y = init_y, t->by += init_y; y < sub_h4; y += t_dim->h, t->by += t_dim->h) { pixel *dst = ((pixel *) f->cur.data[0]) + @@ -1340,12 +1344,12 @@ ac_dump(ac, 4*cbw4, 4*cbh4, "ac"); hex_dump(uv_dst[0], stride, cbw4 * 4, cbh4 * 4, "u-cfl-pred"); hex_dump(uv_dst[1], stride, cbw4 * 4, cbh4 * 4, "v-cfl-pred"); } } else if (b->pal_sz[1]) { - ptrdiff_t uv_dstoff = 4 * ((t->bx >> ss_hor) + - (t->by >> ss_ver) * PXSTRIDE(f->cur.stride[1])); + const ptrdiff_t uv_dstoff = 4 * ((t->bx >> ss_hor) + + (t->by >> ss_ver) * PXSTRIDE(f->cur.stride[1])); const uint16_t (*pal)[8]; const uint8_t *pal_idx; if (f->frame_thread.pass) { assert(ts->frame_thread.pal_idx); pal = f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + @@ -1379,11 +1383,10 @@ ((init_x + 16) >> ss_hor) < cw4 ? 1 : init_y ? 0 : intra_edge_flags & (EDGE_I420_TOP_HAS_RIGHT >> (f->cur.p.layout - 1)); const int uv_sb_has_bl = init_x ? 0 : ((init_y + 16) >> ss_ver) < ch4 ? 1 : intra_edge_flags & (EDGE_I420_LEFT_HAS_BOTTOM >> (f->cur.p.layout - 1)); - const int sub_ch4 = imin(ch4, (init_y + 16) >> ss_ver); const int sub_cw4 = imin(cw4, (init_x + 16) >> ss_hor); for (int pl = 0; pl < 2; pl++) { for (y = init_y >> ss_ver, t->by += init_y; y < sub_ch4; y += uv_t_dim->h, t->by += uv_t_dim->h << ss_ver) { @@ -1515,11 +1518,11 @@ } } } int bytefn(dav1d_recon_b_inter)(Dav1dTileContext *const t, const enum BlockSize bs, - const Av1Block *const b) + const Av1Block *const b) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; const Dav1dDSPContext *const dsp = f->dsp; const int bx4 = t->bx & 31, by4 = t->by & 31; @@ -1607,18 +1610,18 @@ if (!has_chroma) goto skip_inter_chroma_pred; // sub8x8 derivation int is_sub8x8 = bw4 == ss_hor || bh4 == ss_ver; - refmvs *r; + refmvs_block *const *r; if (is_sub8x8) { assert(ss_hor == 1); - r = &f->mvs[t->by * f->b4_stride + t->bx]; - if (bw4 == 1) is_sub8x8 &= r[-1].ref[0] > 0; - if (bh4 == ss_ver) is_sub8x8 &= r[-f->b4_stride].ref[0] > 0; + r = &t->rt.r[(t->by & 31) + 5]; + if (bw4 == 1) is_sub8x8 &= r[0][t->bx - 1].ref.ref[0] > 0; + if (bh4 == ss_ver) is_sub8x8 &= r[-1][t->bx].ref.ref[0] > 0; if (bw4 == 1 && bh4 == ss_ver) - is_sub8x8 &= r[-(1 + f->b4_stride)].ref[0] > 0; + is_sub8x8 &= r[-1][t->bx - 1].ref.ref[0] > 0; } // chroma prediction if (is_sub8x8) { assert(ss_hor == 1); @@ -1626,13 +1629,13 @@ if (bw4 == 1 && bh4 == ss_ver) { for (int pl = 0; pl < 2; pl++) { res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff, NULL, f->cur.stride[1], bw4, bh4, t->bx - 1, t->by - 1, 1 + pl, - r[-(f->b4_stride + 1)].mv[0], - &f->refp[r[-(f->b4_stride + 1)].ref[0] - 1], - r[-(f->b4_stride + 1)].ref[0] - 1, + r[-1][t->bx - 1].mv.mv[0], + &f->refp[r[-1][t->bx - 1].ref.ref[0] - 1], + r[-1][t->bx - 1].ref.ref[0] - 1, f->frame_thread.pass != 2 ? t->tl_4x4_filter : f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx - 1].filter2d); if (res) return res; } v_off = 2 * PXSTRIDE(f->cur.stride[1]); @@ -1642,12 +1645,13 @@ const enum Filter2d left_filter_2d = dav1d_filter_2d[t->l.filter[1][by4]][t->l.filter[0][by4]]; for (int pl = 0; pl < 2; pl++) { res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + v_off, NULL, f->cur.stride[1], bw4, bh4, t->bx - 1, - t->by, 1 + pl, r[-1].mv[0], &f->refp[r[-1].ref[0] - 1], - r[-1].ref[0] - 1, + t->by, 1 + pl, r[0][t->bx - 1].mv.mv[0], + &f->refp[r[0][t->bx - 1].ref.ref[0] - 1], + r[0][t->bx - 1].ref.ref[0] - 1, f->frame_thread.pass != 2 ? left_filter_2d : f->frame_thread.b[(t->by * f->b4_stride) + t->bx - 1].filter2d); if (res) return res; } h_off = 2; @@ -1656,13 +1660,13 @@ const enum Filter2d top_filter_2d = dav1d_filter_2d[t->a->filter[1][bx4]][t->a->filter[0][bx4]]; for (int pl = 0; pl < 2; pl++) { res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + h_off, NULL, f->cur.stride[1], bw4, bh4, t->bx, t->by - 1, - 1 + pl, r[-f->b4_stride].mv[0], - &f->refp[r[-f->b4_stride].ref[0] - 1], - r[-f->b4_stride].ref[0] - 1, + 1 + pl, r[-1][t->bx].mv.mv[0], + &f->refp[r[-1][t->bx].ref.ref[0] - 1], + r[-1][t->bx].ref.ref[0] - 1, f->frame_thread.pass != 2 ? top_filter_2d : f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx].filter2d); if (res) return res; } v_off = 2 * PXSTRIDE(f->cur.stride[1]); @@ -1864,10 +1868,11 @@ return 0; } const TxfmInfo *const uvtx = &dav1d_txfm_dimensions[b->uvtx]; const TxfmInfo *const ytx = &dav1d_txfm_dimensions[b->max_ytx]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; for (int init_y = 0; init_y < bh4; init_y += 16) { for (int init_x = 0; init_x < bw4; init_x += 16) { // coefficient coding & inverse transforms int y_off = !!init_y, y; @@ -1877,11 +1882,11 @@ { int x, x_off = !!init_x; for (x = init_x, t->bx += init_x; x < imin(w4, init_x + 16); x += ytx->w, x_off++) { - read_coef_tree(t, bs, b, b->max_ytx, 0, b->tx_split, + read_coef_tree(t, bs, b, b->max_ytx, 0, tx_split, x_off, y_off, &dst[x * 4]); t->bx += ytx->w; } dst += PXSTRIDE(f->cur.stride[0]) * 4 * ytx->h; t->bx -= x; @@ -1991,11 +1996,11 @@ } const int n_blks = sbsz - 2 * (sby + 1 < sbh); bytefn(dav1d_cdef_brow)(f, f->lf.p, f->lf.mask_ptr, sby * sbsz, imin(sby * sbsz + n_blks, f->bh)); } - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400; for (int pl = 0; pl < 1 + 2 * has_chroma; pl++) { const int ss_ver = pl && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; const int h_start = 8 * !!sby >> ss_ver; const ptrdiff_t dst_stride = f->sr_cur.p.stride[!!pl]; @@ -2006,13 +2011,14 @@ const int ss_hor = pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; const int dst_w = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; const int src_w = (4 * f->bw + ss_hor) >> ss_hor; const int img_h = (f->cur.p.h - sbsz * 4 * sby + ss_ver) >> ss_ver; - f->dsp->mc.resize(dst, dst_stride, src, src_stride, dst_w, src_w, - imin(img_h, h_end) + h_start, f->resize_step[!!pl], - f->resize_start[!!pl] HIGHBD_CALL_SUFFIX); + f->dsp->mc.resize(dst, dst_stride, src, src_stride, dst_w, + imin(img_h, h_end) + h_start, src_w, + f->resize_step[!!pl], f->resize_start[!!pl] + HIGHBD_CALL_SUFFIX); } } if (f->lf.restore_planes) { bytefn(dav1d_lr_sbrow)(f, f->lf.sr_p, sby); } diff --git a/third_party/dav1d/src/ref.c b/third_party/dav1d/src/ref.c --- a/third_party/dav1d/src/ref.c +++ b/third_party/dav1d/src/ref.c @@ -35,29 +35,25 @@ assert(data == user_data); dav1d_free_aligned(user_data); } Dav1dRef *dav1d_ref_create(const size_t size) { - Dav1dRef *res; void *data = dav1d_alloc_aligned(size, 32); - if (!data) { - return NULL; - } + if (!data) return NULL; - res = dav1d_ref_wrap(data, default_free_callback, data); - if (!res) { + Dav1dRef *const res = dav1d_ref_wrap(data, default_free_callback, data); + if (res) + res->data = data; + else dav1d_free_aligned(data); - } else { - res->data = data; - } return res; } Dav1dRef *dav1d_ref_wrap(const uint8_t *const ptr, void (*free_callback)(const uint8_t *data, void *user_data), - void *user_data) + void *const user_data) { Dav1dRef *res = malloc(sizeof(Dav1dRef)); if (!res) return NULL; res->data = NULL; diff --git a/third_party/dav1d/src/ref_mvs.h b/third_party/dav1d/src/ref_mvs.h deleted file mode 100644 --- a/third_party/dav1d/src/ref_mvs.h +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef DAV1D_SRC_REF_MVS_H -#define DAV1D_SRC_REF_MVS_H - -#include - -#include "src/levels.h" -#include "src/tables.h" - -typedef struct refmvs { - mv mv[2]; - int8_t ref[2]; // [0] = 0: intra=1, [1] = -1: comp=0 - int8_t mode, sb_type; -} refmvs; - -typedef struct candidate_mv { - mv this_mv; - mv comp_mv; - int weight; -} candidate_mv; - -typedef struct AV1_COMMON AV1_COMMON; - -// call once per frame thread -AV1_COMMON *dav1d_alloc_ref_mv_common(void); -void dav1d_free_ref_mv_common(AV1_COMMON *cm); - -// call once per frame -int dav1d_init_ref_mv_common(AV1_COMMON *cm, int w8, int h8, - ptrdiff_t stride, int allow_sb128, - refmvs *cur, refmvs *ref_mvs[7], - unsigned cur_poc, - const unsigned ref_poc[7], - const unsigned ref_ref_poc[7][7], - const Dav1dWarpedMotionParams gmv[7], - int allow_hp, int force_int_mv, - int allow_ref_frame_mvs, int order_hint); - -// call for start of each sbrow per tile -void dav1d_init_ref_mv_tile_row(AV1_COMMON *cm, - int tile_col_start4, int tile_col_end4, - int row_start4, int row_end4); - -// call for each block -void dav1d_find_ref_mvs(candidate_mv *mvstack, int *cnt, mv (*mvlist)[2], - int *ctx, int refidx[2], int w4, int h4, - enum BlockSize bs, enum BlockPartition bp, - int by4, int bx4, int tile_col_start4, - int tile_col_end4, int tile_row_start4, - int tile_row_end4, AV1_COMMON *cm); - -extern const uint8_t dav1d_bs_to_sbtype[]; -extern const uint8_t dav1d_sbtype_to_bs[]; -static inline void splat_oneref_mv(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, - const enum InterPredMode mode, - const int ref, const mv mv, - const int is_interintra) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - const refmvs tmpl = (refmvs) { - .ref = { ref + 1, is_interintra ? 0 : -1 }, - .mv = { mv }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = N_INTRA_PRED_MODES + mode, - }; - do { - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - r += stride; - } while (--bh4); -} - -static inline void splat_intrabc_mv(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, const mv mv) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - const refmvs tmpl = (refmvs) { - .ref = { 0, -1 }, - .mv = { mv }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = DC_PRED, - }; - do { - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - r += stride; - } while (--bh4); -} - -static inline void splat_tworef_mv(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, - const enum CompInterPredMode mode, - const int ref1, const int ref2, - const mv mv1, const mv mv2) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - const refmvs tmpl = (refmvs) { - .ref = { ref1 + 1, ref2 + 1 }, - .mv = { mv1, mv2 }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = N_INTRA_PRED_MODES + N_INTER_PRED_MODES + mode, - }; - do { - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - r += stride; - } while (--bh4); -} - -static inline void splat_intraref(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, - const enum IntraPredMode mode) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - do { - int x; - - for (x = 0; x < bw4; x++) - r[x] = (refmvs) { - .ref = { 0, -1 }, - .mv = { [0] = { .y = -0x8000, .x = -0x8000 }, }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = mode, - }; - r += stride; - } while (--bh4); -} - -static inline void fix_mv_precision(const Dav1dFrameHeader *const hdr, - mv *const mv) -{ - if (hdr->force_integer_mv) { - const int xmod = mv->x & 7; - mv->x &= ~7; - mv->x += (xmod > 4 - (mv->x < 0)) << 3; - const int ymod = mv->y & 7; - mv->y &= ~7; - mv->y += (ymod > 4 - (mv->y < 0)) << 3; - } else if (!hdr->hp) { - if (mv->x & 1) { - if (mv->x < 0) mv->x++; - else mv->x--; - } - if (mv->y & 1) { - if (mv->y < 0) mv->y++; - else mv->y--; - } - } -} - -#endif /* DAV1D_SRC_REF_MVS_H */ diff --git a/third_party/dav1d/src/ref_mvs.c b/third_party/dav1d/src/ref_mvs.c deleted file mode 100644 --- a/third_party/dav1d/src/ref_mvs.c +++ /dev/null @@ -1,2107 +0,0 @@ -/* - * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -/* - * Changes made compared to libaom version: - * - we disable TMV and enable MV_COMPRESS so that the - * input array for prev_frames can be at 4x4 instead of - * 8x8 resolution, and therefore shared between cur_frame - * and prev_frame. To make enc/dec behave consistent, we - * also make this change around line 2580: -#if 0 - AOMMIN(((mi_row >> 1) << 1) + 1 + (((xd->n8_h - 1) >> 1) << 1), - mi_row_end - 1) * - prev_frame_mvs_stride + - AOMMIN(((mi_col >> 1) << 1) + 1 + (((xd->n8_w - 1) >> 1) << 1), - mi_col_end - 1) -#else - (((mi_row >> 1) << 1) + 1) * prev_frame_mvs_stride + - (((mi_col >> 1) << 1) + 1) -#endif - * and the same change (swap mi_cols from prev_frame.mv_stride) on line 2407 - * - we disable rect-block overhanging edge inclusion (see - * line 2642): - if (num_8x8_blocks_wide == num_8x8_blocks_high || 1) { - mv_ref_search[5].row = -1; - mv_ref_search[5].col = 0; - mv_ref_search[6].row = 0; - mv_ref_search[6].col = -1; - } else { - mv_ref_search[5].row = -1; - mv_ref_search[5].col = num_8x8_blocks_wide; - mv_ref_search[6].row = num_8x8_blocks_high; - mv_ref_search[6].col = -1; - } - * Note that this is a bitstream change and needs the same - * change on the decoder side also. - * - we change xd->mi to be a pointer instead of a double ptr. - */ - -#include "config.h" - -#include -#include -#include -#include -#include -#include - -#include "dav1d/common.h" - -#include "common/intops.h" - -#define av1_zero(a) memset(a, 0, sizeof(a)) - -#define ATTRIBUTE_PACKED -#define INLINE inline -#define IMPLIES(a, b) (!(a) || (b)) // Logical 'a implies b' (or 'a -> b') - -#define ROUND_POWER_OF_TWO(value, n) (((value) + (((1 << (n)) >> 1))) >> (n)) -#define ROUND_POWER_OF_TWO_SIGNED(value, n) \ - (((value) < 0) ? -ROUND_POWER_OF_TWO(-(value), (n)) \ - : ROUND_POWER_OF_TWO((value), (n))) -#define NELEMENTS(x) (int)(sizeof(x) / sizeof(x[0])) - -#define MAX_MV_REF_CANDIDATES 2 - -#define MAX_REF_MV_STACK_SIZE 8 -#define REF_CAT_LEVEL 640 - -#define FRAME_OFFSET_BITS 5 -#define MAX_FRAME_DISTANCE ((1 << FRAME_OFFSET_BITS) - 1) -#define INVALID_MV 0x80008000 - -#define COMP_NEWMV_CTXS 5 -#define REFMV_OFFSET 4 -#define REFMV_CTX_MASK ((1 << (8 - REFMV_OFFSET)) - 1) - -#define MV_IN_USE_BITS 14 -#define MV_UPP (1 << MV_IN_USE_BITS) -#define MV_LOW (-(1 << MV_IN_USE_BITS)) - -typedef struct MV { - int16_t row; - int16_t col; -} MV; -typedef union int_mv { - uint32_t as_int; - MV as_mv; -} int_mv; -typedef int8_t MV_REFERENCE_FRAME; -#define MFMV_STACK_SIZE 3 -typedef struct { - int_mv mfmv0; - uint8_t ref_frame_offset; -} TPL_MV_REF; -typedef struct { - int_mv mv[2]; - MV_REFERENCE_FRAME ref_frame[2]; - int8_t mode, sb_type; -} MV_REF; -#define MB_MODE_INFO MV_REF - -#define AOMMAX(a,b) ((a)>(b)?(a):(b)) -#define AOMMIN(a,b) ((a)<(b)?(a):(b)) - -typedef struct candidate_mv { - int_mv this_mv; - int_mv comp_mv; - int weight; -} CANDIDATE_MV; -#define NONE_FRAME -1 -#define INTRA_FRAME 0 -#define LAST_FRAME 1 - -#define LAST2_FRAME 2 -#define LAST3_FRAME 3 -#define GOLDEN_FRAME 4 -#define BWDREF_FRAME 5 -#define ALTREF2_FRAME 6 -#define ALTREF_FRAME 7 -#define LAST_REF_FRAMES (LAST3_FRAME - LAST_FRAME + 1) - -#define INTER_REFS_PER_FRAME (ALTREF_FRAME - LAST_FRAME + 1) -#define TOTAL_REFS_PER_FRAME (ALTREF_FRAME - INTRA_FRAME + 1) - -#define FWD_REFS (GOLDEN_FRAME - LAST_FRAME + 1) -#define FWD_RF_OFFSET(ref) (ref - LAST_FRAME) -#define BWD_REFS (ALTREF_FRAME - BWDREF_FRAME + 1) -#define BWD_RF_OFFSET(ref) (ref - BWDREF_FRAME) -#define FWD_REFS (GOLDEN_FRAME - LAST_FRAME + 1) -#define SINGLE_REFS (FWD_REFS + BWD_REFS) -typedef enum ATTRIBUTE_PACKED { - LAST_LAST2_FRAMES, // { LAST_FRAME, LAST2_FRAME } - LAST_LAST3_FRAMES, // { LAST_FRAME, LAST3_FRAME } - LAST_GOLDEN_FRAMES, // { LAST_FRAME, GOLDEN_FRAME } - BWDREF_ALTREF_FRAMES, // { BWDREF_FRAME, ALTREF_FRAME } - LAST2_LAST3_FRAMES, // { LAST2_FRAME, LAST3_FRAME } - LAST2_GOLDEN_FRAMES, // { LAST2_FRAME, GOLDEN_FRAME } - LAST3_GOLDEN_FRAMES, // { LAST3_FRAME, GOLDEN_FRAME } - BWDREF_ALTREF2_FRAMES, // { BWDREF_FRAME, ALTREF2_FRAME } - ALTREF2_ALTREF_FRAMES, // { ALTREF2_FRAME, ALTREF_FRAME } - TOTAL_UNIDIR_COMP_REFS, - // NOTE: UNIDIR_COMP_REFS is the number of uni-directional reference pairs - // that are explicitly signaled. - UNIDIR_COMP_REFS = BWDREF_ALTREF_FRAMES + 1, -} UNIDIR_COMP_REF; -#define TOTAL_COMP_REFS (FWD_REFS * BWD_REFS + TOTAL_UNIDIR_COMP_REFS) -#define MODE_CTX_REF_FRAMES (TOTAL_REFS_PER_FRAME + TOTAL_COMP_REFS) - -#define GLOBALMV_OFFSET 3 -#define NEWMV_CTX_MASK ((1 << GLOBALMV_OFFSET) - 1) -#define GLOBALMV_CTX_MASK ((1 << (REFMV_OFFSET - GLOBALMV_OFFSET)) - 1) -#define MI_SIZE_LOG2 2 -#define MI_SIZE (1 << MI_SIZE_LOG2) -#define MAX_SB_SIZE_LOG2 7 -#define MAX_MIB_SIZE_LOG2 (MAX_SB_SIZE_LOG2 - MI_SIZE_LOG2) -#define MIN_MIB_SIZE_LOG2 (MIN_SB_SIZE_LOG2 - MI_SIZE_LOG2) -#define MAX_MIB_SIZE (1 << MAX_MIB_SIZE_LOG2) -#define MI_SIZE_64X64 (64 >> MI_SIZE_LOG2) -#define MI_SIZE_128X128 (128 >> MI_SIZE_LOG2) -#define REFMV_OFFSET 4 - -typedef enum ATTRIBUTE_PACKED { - BLOCK_4X4, - BLOCK_4X8, - BLOCK_8X4, - BLOCK_8X8, - BLOCK_8X16, - BLOCK_16X8, - BLOCK_16X16, - BLOCK_16X32, - BLOCK_32X16, - BLOCK_32X32, - BLOCK_32X64, - BLOCK_64X32, - BLOCK_64X64, - BLOCK_64X128, - BLOCK_128X64, - BLOCK_128X128, - BLOCK_4X16, - BLOCK_16X4, - BLOCK_8X32, - BLOCK_32X8, - BLOCK_16X64, - BLOCK_64X16, - BLOCK_32X128, - BLOCK_128X32, - BLOCK_SIZES_ALL, - BLOCK_SIZES = BLOCK_4X16, - BLOCK_INVALID = 255, - BLOCK_LARGEST = (BLOCK_SIZES - 1) -} BLOCK_SIZE; - -typedef enum ATTRIBUTE_PACKED { - PARTITION_NONE, - PARTITION_HORZ, - PARTITION_VERT, - PARTITION_SPLIT, - PARTITION_HORZ_A, // HORZ split and the top partition is split again - PARTITION_HORZ_B, // HORZ split and the bottom partition is split again - PARTITION_VERT_A, // VERT split and the left partition is split again - PARTITION_VERT_B, // VERT split and the right partition is split again - PARTITION_HORZ_4, // 4:1 horizontal partition - PARTITION_VERT_4, // 4:1 vertical partition - EXT_PARTITION_TYPES, - PARTITION_TYPES = PARTITION_SPLIT + 1, - PARTITION_INVALID = 255 -} PARTITION_TYPE; -typedef struct CUR_MODE_INFO { - PARTITION_TYPE partition; -} CUR_MODE_INFO ; - -typedef enum ATTRIBUTE_PACKED { - DC_PRED, // Average of above and left pixels - V_PRED, // Vertical - H_PRED, // Horizontal - D45_PRED, // Directional 45 deg = round(arctan(1/1) * 180/pi) - D135_PRED, // Directional 135 deg = 180 - 45 - D117_PRED, // Directional 117 deg = 180 - 63 - D153_PRED, // Directional 153 deg = 180 - 27 - D207_PRED, // Directional 207 deg = 180 + 27 - D63_PRED, // Directional 63 deg = round(arctan(2/1) * 180/pi) - SMOOTH_PRED, // Combination of horizontal and vertical interpolation - SMOOTH_V_PRED, // Vertical interpolation - SMOOTH_H_PRED, // Horizontal interpolation - PAETH_PRED, // Predict from the direction of smallest gradient - NEARESTMV, - NEARMV, - GLOBALMV, - NEWMV, - // Compound ref compound modes - NEAREST_NEARESTMV, - NEAR_NEARMV, - NEAREST_NEWMV, - NEW_NEARESTMV, - NEAR_NEWMV, - NEW_NEARMV, - GLOBAL_GLOBALMV, - NEW_NEWMV, - MB_MODE_COUNT, - INTRA_MODES = PAETH_PRED + 1, // PAETH_PRED has to be the last intra mode. - INTRA_INVALID = MB_MODE_COUNT // For uv_mode in inter blocks -} PREDICTION_MODE; -typedef enum { - IDENTITY = 0, // identity transformation, 0-parameter - TRANSLATION = 1, // translational motion 2-parameter - ROTZOOM = 2, // simplified affine with rotation + zoom only, 4-parameter - AFFINE = 3, // affine, 6-parameter - TRANS_TYPES, -} TransformationType; - -#define LEAST_SQUARES_SAMPLES_MAX_BITS 3 -#define LEAST_SQUARES_SAMPLES_MAX (1 << LEAST_SQUARES_SAMPLES_MAX_BITS) -#define SAMPLES_ARRAY_SIZE (LEAST_SQUARES_SAMPLES_MAX * 2) - -static const uint8_t mi_size_wide[BLOCK_SIZES_ALL] = { - 1, 1, 2, 2, 2, 4, 4, 4, 8, 8, 8, 16, 16, - 16, 32, 32, 1, 4, 2, 8, 4, 16, 8, 32 -}; -static const uint8_t mi_size_high[BLOCK_SIZES_ALL] = { - 1, 2, 1, 2, 4, 2, 4, 8, 4, 8, 16, 8, 16, - 32, 16, 32, 4, 1, 8, 2, 16, 4, 32, 8 -}; - -static const uint8_t block_size_wide[BLOCK_SIZES_ALL] = { - 4, 4, - 8, 8, - 8, 16, - 16, 16, - 32, 32, - 32, 64, - 64, 64, 128, 128, 4, - 16, 8, - 32, 16, - 64, 32, 128 -}; - -static const uint8_t block_size_high[BLOCK_SIZES_ALL] = { - 4, 8, - 4, 8, - 16, 8, - 16, 32, - 16, 32, - 64, 32, - 64, 128, 64, 128, 16, - 4, 32, - 8, 64, - 16, 128, 32 -}; - -static INLINE int is_global_mv_block(const MB_MODE_INFO *const mbmi, - TransformationType type) { - const PREDICTION_MODE mode = mbmi->mode; - const BLOCK_SIZE bsize = mbmi->sb_type; - const int block_size_allowed = - AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8; - return block_size_allowed && type > TRANSLATION && - (mode == GLOBALMV || mode == GLOBAL_GLOBALMV); -} - -typedef struct { - TransformationType wmtype; - int32_t wmmat[6]; - int16_t alpha, beta, gamma, delta; -} Dav1dWarpedMotionParams; - -#define REF_FRAMES_LOG2 3 -#define REF_FRAMES (1 << REF_FRAMES_LOG2) -#define FRAME_BUFFERS (REF_FRAMES + 7) -typedef struct { - - unsigned int cur_frame_offset; - unsigned int ref_frame_offset[INTER_REFS_PER_FRAME]; - - MV_REF *mvs; - ptrdiff_t mv_stride; - int mi_rows; - int mi_cols; - uint8_t intra_only; -} RefCntBuffer; - -#define INVALID_IDX -1 // Invalid buffer index. -typedef struct TileInfo { - int mi_row_start, mi_row_end; - int mi_col_start, mi_col_end; - int tg_horz_boundary; -} TileInfo; -typedef struct macroblockd { - TileInfo tile; - int mi_stride; - - CUR_MODE_INFO cur_mi; - MB_MODE_INFO *mi; - int up_available; - int left_available; - /* Distance of MB away from frame edges in subpixels (1/8th pixel) */ - int mb_to_left_edge; - int mb_to_right_edge; - int mb_to_top_edge; - int mb_to_bottom_edge; - // block dimension in the unit of mode_info. - uint8_t n8_w, n8_h; - uint8_t is_sec_rect; - -} MACROBLOCKD; -typedef struct RefBuffer { - int idx; // frame buf idx -} RefBuffer; -typedef struct BufferPool { - RefCntBuffer frame_bufs[FRAME_BUFFERS]; -} BufferPool; -typedef struct AV1Common { - - // TODO(hkuang): Combine this with cur_buf in macroblockd. - RefCntBuffer cur_frame; - - // Each Inter frame can reference INTER_REFS_PER_FRAME buffers - RefBuffer frame_refs[INTER_REFS_PER_FRAME]; - - int allow_high_precision_mv; - int cur_frame_force_integer_mv; // 0 the default in AOM, 1 only integer - int mi_rows; - int mi_cols; - int mi_stride; - - // Whether to use previous frame's motion vectors for prediction. - int allow_ref_frame_mvs; - - int ref_frame_sign_bias[TOTAL_REFS_PER_FRAME]; /* Two state 0, 1 */ - int frame_parallel_decode; // frame-based threading. - - unsigned int frame_offset; - - // External BufferPool passed from outside. - BufferPool buffer_pool; - - Dav1dWarpedMotionParams global_motion[TOTAL_REFS_PER_FRAME]; - struct { - BLOCK_SIZE sb_size; - int enable_order_hint; - int order_hint_bits_minus1; - } seq_params; - TPL_MV_REF *tpl_mvs; - // TODO(jingning): This can be combined with sign_bias later. - int8_t ref_frame_side[TOTAL_REFS_PER_FRAME]; - - int ref_buf_idx[INTER_REFS_PER_FRAME]; - int ref_order_hint[INTER_REFS_PER_FRAME]; -} AV1_COMMON; - -static INLINE void integer_mv_precision(MV *mv) { - int mod = (mv->row % 8); - if (mod != 0) { - mv->row -= mod; - if (abs(mod) > 4) { - if (mod > 0) { - mv->row += 8; - } else { - mv->row -= 8; - } - } - } - - mod = (mv->col % 8); - if (mod != 0) { - mv->col -= mod; - if (abs(mod) > 4) { - if (mod > 0) { - mv->col += 8; - } else { - mv->col -= 8; - } - } - } -} - -static INLINE int clamp(int value, int low, int high) { - return value < low ? low : (value > high ? high : value); -} - -static INLINE void clamp_mv(MV *mv, int min_col, int max_col, int min_row, - int max_row) { - mv->col = clamp(mv->col, min_col, max_col); - mv->row = clamp(mv->row, min_row, max_row); -} - -static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) { - return mbmi->ref_frame[0] == INTRA_FRAME && mbmi->mv[0].as_mv.row != -0x8000; - //return mbmi->use_intrabc; -} - -static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) { - if (is_intrabc_block(mbmi)) return 1; - return mbmi->ref_frame[0] > INTRA_FRAME; -} - -static INLINE MV_REFERENCE_FRAME comp_ref0(int ref_idx) { - static const MV_REFERENCE_FRAME lut[] = { - LAST_FRAME, // LAST_LAST2_FRAMES, - LAST_FRAME, // LAST_LAST3_FRAMES, - LAST_FRAME, // LAST_GOLDEN_FRAMES, - BWDREF_FRAME, // BWDREF_ALTREF_FRAMES, - LAST2_FRAME, // LAST2_LAST3_FRAMES - LAST2_FRAME, // LAST2_GOLDEN_FRAMES, - LAST3_FRAME, // LAST3_GOLDEN_FRAMES, - BWDREF_FRAME, // BWDREF_ALTREF2_FRAMES, - ALTREF2_FRAME, // ALTREF2_ALTREF_FRAMES, - }; - assert(NELEMENTS(lut) == TOTAL_UNIDIR_COMP_REFS); - return lut[ref_idx]; -} - -static INLINE MV_REFERENCE_FRAME comp_ref1(int ref_idx) { - static const MV_REFERENCE_FRAME lut[] = { - LAST2_FRAME, // LAST_LAST2_FRAMES, - LAST3_FRAME, // LAST_LAST3_FRAMES, - GOLDEN_FRAME, // LAST_GOLDEN_FRAMES, - ALTREF_FRAME, // BWDREF_ALTREF_FRAMES, - LAST3_FRAME, // LAST2_LAST3_FRAMES - GOLDEN_FRAME, // LAST2_GOLDEN_FRAMES, - GOLDEN_FRAME, // LAST3_GOLDEN_FRAMES, - ALTREF2_FRAME, // BWDREF_ALTREF2_FRAMES, - ALTREF_FRAME, // ALTREF2_ALTREF_FRAMES, - }; - assert(NELEMENTS(lut) == TOTAL_UNIDIR_COMP_REFS); - return lut[ref_idx]; -} - -#define WARPEDMODEL_PREC_BITS 16 -#define GM_TRANS_ONLY_PREC_DIFF (WARPEDMODEL_PREC_BITS - 3) -#define WARPEDMODEL_ROW3HOMO_PREC_BITS 16 - -static INLINE int convert_to_trans_prec(int allow_hp, int coor) { - if (allow_hp) - return ROUND_POWER_OF_TWO_SIGNED(coor, WARPEDMODEL_PREC_BITS - 3); - else - return ROUND_POWER_OF_TWO_SIGNED(coor, WARPEDMODEL_PREC_BITS - 2) * 2; -} - -static INLINE int block_center_x(int mi_col, BLOCK_SIZE bs) { - const int bw = block_size_wide[bs]; - return mi_col * MI_SIZE + bw / 2 - 1; -} - -static INLINE int block_center_y(int mi_row, BLOCK_SIZE bs) { - const int bh = block_size_high[bs]; - return mi_row * MI_SIZE + bh / 2 - 1; -} - -// Convert a global motion vector into a motion vector at the centre of the -// given block. -// -// The resulting motion vector will have three fractional bits of precision. If -// allow_hp is zero, the bottom bit will always be zero. If CONFIG_AMVR and -// is_integer is true, the bottom three bits will be zero (so the motion vector -// represents an integer) -static INLINE int_mv gm_get_motion_vector(const Dav1dWarpedMotionParams *gm, - int allow_hp, BLOCK_SIZE bsize, - int mi_col, int mi_row, - int is_integer) { - int_mv res; - const int32_t *mat = gm->wmmat; - int x, y, tx, ty; - - if (gm->wmtype == TRANSLATION) { - // All global motion vectors are stored with WARPEDMODEL_PREC_BITS (16) - // bits of fractional precision. The offset for a translation is stored in - // entries 0 and 1. For translations, all but the top three (two if - // cm->allow_high_precision_mv is false) fractional bits are always zero. - // - // After the right shifts, there are 3 fractional bits of precision. If - // allow_hp is false, the bottom bit is always zero (so we don't need a - // call to convert_to_trans_prec here) - res.as_mv.row = gm->wmmat[0] >> GM_TRANS_ONLY_PREC_DIFF; - res.as_mv.col = gm->wmmat[1] >> GM_TRANS_ONLY_PREC_DIFF; - assert(IMPLIES(1 & (res.as_mv.row | res.as_mv.col), allow_hp)); - if (is_integer) { - integer_mv_precision(&res.as_mv); - } - return res; - } - - x = block_center_x(mi_col, bsize); - y = block_center_y(mi_row, bsize); - - if (gm->wmtype == ROTZOOM) { - assert(gm->wmmat[5] == gm->wmmat[2]); - assert(gm->wmmat[4] == -gm->wmmat[3]); - } - if (gm->wmtype > AFFINE) { - int xc = (int)((int64_t)mat[2] * x + (int64_t)mat[3] * y + mat[0]); - int yc = (int)((int64_t)mat[4] * x + (int64_t)mat[5] * y + mat[1]); - const int Z = (int)((int64_t)mat[6] * x + (int64_t)mat[7] * y + - (1 << WARPEDMODEL_ROW3HOMO_PREC_BITS)); - xc *= 1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS - WARPEDMODEL_PREC_BITS); - yc *= 1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS - WARPEDMODEL_PREC_BITS); - xc = (int)(xc > 0 ? ((int64_t)xc + Z / 2) / Z : ((int64_t)xc - Z / 2) / Z); - yc = (int)(yc > 0 ? ((int64_t)yc + Z / 2) / Z : ((int64_t)yc - Z / 2) / Z); - tx = convert_to_trans_prec(allow_hp, xc) - (x << 3); - ty = convert_to_trans_prec(allow_hp, yc) - (y << 3); - } else { - const int xc = - (mat[2] - (1 << WARPEDMODEL_PREC_BITS)) * x + mat[3] * y + mat[0]; - const int yc = - mat[4] * x + (mat[5] - (1 << WARPEDMODEL_PREC_BITS)) * y + mat[1]; - tx = convert_to_trans_prec(allow_hp, xc); - ty = convert_to_trans_prec(allow_hp, yc); - } - - res.as_mv.row = ty; - res.as_mv.col = tx; - - if (is_integer) { - integer_mv_precision(&res.as_mv); - } - return res; -} - -static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) { - return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV || - mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV); -} - -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ -#ifndef AV1_COMMON_MVREF_COMMON_H_ -#define AV1_COMMON_MVREF_COMMON_H_ - -//#include "av1/common/onyxc_int.h" -//#include "av1/common/blockd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define MVREF_ROW_COLS 3 - -// Set the upper limit of the motion vector component magnitude. -// This would make a motion vector fit in 26 bits. Plus 3 bits for the -// reference frame index. A tuple of motion vector can hence be stored within -// 32 bit range for efficient load/store operations. -#define REFMVS_LIMIT ((1 << 12) - 1) - -typedef struct position { - int row; - int col; -} POSITION; - -// clamp_mv_ref -#define MV_BORDER (16 << 3) // Allow 16 pels in 1/8th pel units - -static INLINE int get_relative_dist(const AV1_COMMON *cm, int a, int b) { - if (!cm->seq_params.enable_order_hint) return 0; - - const int bits = cm->seq_params.order_hint_bits_minus1 + 1; - - assert(bits >= 1); - assert(a >= 0 && a < (1 << bits)); - assert(b >= 0 && b < (1 << bits)); - - int diff = a - b; - int m = 1 << (bits - 1); - diff = (diff & (m - 1)) - (diff & m); - return diff; -} - -static INLINE void clamp_mv_ref(MV *mv, int bw, int bh, const MACROBLOCKD *xd) { - clamp_mv(mv, xd->mb_to_left_edge - bw * 8 - MV_BORDER, - xd->mb_to_right_edge + bw * 8 + MV_BORDER, - xd->mb_to_top_edge - bh * 8 - MV_BORDER, - xd->mb_to_bottom_edge + bh * 8 + MV_BORDER); -} - -// This function returns either the appropriate sub block or block's mv -// on whether the block_size < 8x8 and we have check_sub_blocks set. -static INLINE int_mv get_sub_block_mv(const MB_MODE_INFO *candidate, - int which_mv, int search_col) { - (void)search_col; - return candidate->mv[which_mv]; -} - -// Checks that the given mi_row, mi_col and search point -// are inside the borders of the tile. -static INLINE int is_inside(const TileInfo *const tile, int mi_col, int mi_row, - int mi_rows, const POSITION *mi_pos) { - const int dependent_horz_tile_flag = 0; - if (dependent_horz_tile_flag && !tile->tg_horz_boundary) { - return !(mi_row + mi_pos->row < 0 || - mi_col + mi_pos->col < tile->mi_col_start || - mi_row + mi_pos->row >= mi_rows || - mi_col + mi_pos->col >= tile->mi_col_end); - } else { - return !(mi_row + mi_pos->row < tile->mi_row_start || - mi_col + mi_pos->col < tile->mi_col_start || - mi_row + mi_pos->row >= tile->mi_row_end || - mi_col + mi_pos->col >= tile->mi_col_end); - } -} - -static INLINE int find_valid_row_offset(const TileInfo *const tile, int mi_row, - int mi_rows, int row_offset) { - const int dependent_horz_tile_flag = 0; - if (dependent_horz_tile_flag && !tile->tg_horz_boundary) - return clamp(row_offset, -mi_row, mi_rows - mi_row - 1); - else - return clamp(row_offset, tile->mi_row_start - mi_row, - tile->mi_row_end - mi_row - 1); -} - -static INLINE int find_valid_col_offset(const TileInfo *const tile, int mi_col, - int col_offset) { - return clamp(col_offset, tile->mi_col_start - mi_col, - tile->mi_col_end - mi_col - 1); -} - -static INLINE void lower_mv_precision(MV *mv, int allow_hp, - int is_integer) { - if (is_integer) { - integer_mv_precision(mv); - } else { - if (!allow_hp) { - if (mv->row & 1) mv->row += (mv->row > 0 ? -1 : 1); - if (mv->col & 1) mv->col += (mv->col > 0 ? -1 : 1); - } - } -} - -static INLINE int8_t get_uni_comp_ref_idx(const MV_REFERENCE_FRAME *const rf) { - // Single ref pred - if (rf[1] <= INTRA_FRAME) return -1; - - // Bi-directional comp ref pred - if ((rf[0] < BWDREF_FRAME) && (rf[1] >= BWDREF_FRAME)) return -1; - - for (int8_t ref_idx = 0; ref_idx < TOTAL_UNIDIR_COMP_REFS; ++ref_idx) { - if (rf[0] == comp_ref0(ref_idx) && rf[1] == comp_ref1(ref_idx)) - return ref_idx; - } - return -1; -} - -static INLINE int8_t av1_ref_frame_type(const MV_REFERENCE_FRAME *const rf) { - if (rf[1] > INTRA_FRAME) { - const int8_t uni_comp_ref_idx = get_uni_comp_ref_idx(rf); - if (uni_comp_ref_idx >= 0) { - assert((REF_FRAMES + FWD_REFS * BWD_REFS + uni_comp_ref_idx) < - MODE_CTX_REF_FRAMES); - return REF_FRAMES + FWD_REFS * BWD_REFS + uni_comp_ref_idx; - } else { - return REF_FRAMES + FWD_RF_OFFSET(rf[0]) + - BWD_RF_OFFSET(rf[1]) * FWD_REFS; - } - } - - return rf[0]; -} - -// clang-format off -static MV_REFERENCE_FRAME ref_frame_map[TOTAL_COMP_REFS][2] = { - { LAST_FRAME, BWDREF_FRAME }, { LAST2_FRAME, BWDREF_FRAME }, - { LAST3_FRAME, BWDREF_FRAME }, { GOLDEN_FRAME, BWDREF_FRAME }, - - { LAST_FRAME, ALTREF2_FRAME }, { LAST2_FRAME, ALTREF2_FRAME }, - { LAST3_FRAME, ALTREF2_FRAME }, { GOLDEN_FRAME, ALTREF2_FRAME }, - - { LAST_FRAME, ALTREF_FRAME }, { LAST2_FRAME, ALTREF_FRAME }, - { LAST3_FRAME, ALTREF_FRAME }, { GOLDEN_FRAME, ALTREF_FRAME }, - - { LAST_FRAME, LAST2_FRAME }, { LAST_FRAME, LAST3_FRAME }, - { LAST_FRAME, GOLDEN_FRAME }, { BWDREF_FRAME, ALTREF_FRAME }, - - // NOTE: Following reference frame pairs are not supported to be explicitly - // signalled, but they are possibly chosen by the use of skip_mode, - // which may use the most recent one-sided reference frame pair. - { LAST2_FRAME, LAST3_FRAME }, { LAST2_FRAME, GOLDEN_FRAME }, - { LAST3_FRAME, GOLDEN_FRAME }, {BWDREF_FRAME, ALTREF2_FRAME}, - { ALTREF2_FRAME, ALTREF_FRAME } -}; -// clang-format on - -static INLINE void av1_set_ref_frame(MV_REFERENCE_FRAME *rf, - int8_t ref_frame_type) { - if (ref_frame_type >= REF_FRAMES) { - rf[0] = ref_frame_map[ref_frame_type - REF_FRAMES][0]; - rf[1] = ref_frame_map[ref_frame_type - REF_FRAMES][1]; - } else { - rf[0] = ref_frame_type; - rf[1] = NONE_FRAME; - assert(ref_frame_type > NONE_FRAME); - } -} - -static uint16_t compound_mode_ctx_map[3][COMP_NEWMV_CTXS] = { - { 0, 1, 1, 1, 1 }, - { 1, 2, 3, 4, 4 }, - { 4, 4, 5, 6, 7 }, -}; - -static INLINE int16_t av1_mode_context_analyzer( - const int16_t *const mode_context, const MV_REFERENCE_FRAME *const rf) { - const int8_t ref_frame = av1_ref_frame_type(rf); - - if (rf[1] <= INTRA_FRAME) return mode_context[ref_frame]; - - const int16_t newmv_ctx = mode_context[ref_frame] & NEWMV_CTX_MASK; - const int16_t refmv_ctx = - (mode_context[ref_frame] >> REFMV_OFFSET) & REFMV_CTX_MASK; - - const int16_t comp_ctx = compound_mode_ctx_map[refmv_ctx >> 1][AOMMIN( - newmv_ctx, COMP_NEWMV_CTXS - 1)]; - return comp_ctx; -} - -#define INTRABC_DELAY_PIXELS 256 // Delay of 256 pixels -#define INTRABC_DELAY_SB64 (INTRABC_DELAY_PIXELS / 64) -#define USE_WAVE_FRONT 1 // Use only top left area of frame for reference. - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // AV1_COMMON_MVREF_COMMON_H_ - -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include - -//#include "av1/common/mvref_common.h" -//#include "av1/common/warped_motion.h" - -// Although we assign 32 bit integers, all the values are strictly under 14 -// bits. -static int div_mult[32] = { 0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, - 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, - 1024, 963, 910, 862, 819, 780, 744, 712, - 682, 655, 630, 606, 585, 564, 546, 528 }; - -// TODO(jingning): Consider the use of lookup table for (num / den) -// altogether. -static void get_mv_projection(MV *output, MV ref, int num, int den) { - den = AOMMIN(den, MAX_FRAME_DISTANCE); - num = num > 0 ? AOMMIN(num, MAX_FRAME_DISTANCE) - : AOMMAX(num, -MAX_FRAME_DISTANCE); - int mv_row = ROUND_POWER_OF_TWO_SIGNED(ref.row * num * div_mult[den], 14); - int mv_col = ROUND_POWER_OF_TWO_SIGNED(ref.col * num * div_mult[den], 14); - const int clamp_max = MV_UPP - 1; - const int clamp_min = MV_LOW + 1; - output->row = (int16_t)clamp(mv_row, clamp_min, clamp_max); - output->col = (int16_t)clamp(mv_col, clamp_min, clamp_max); -} - -static void add_ref_mv_candidate( - const MB_MODE_INFO *const candidate, const MV_REFERENCE_FRAME rf[2], - uint8_t *refmv_count, uint8_t *ref_match_count, uint8_t *newmv_count, - CANDIDATE_MV *ref_mv_stack, int_mv *gm_mv_candidates, - const Dav1dWarpedMotionParams *gm_params, int col, int weight) { - if (!is_inter_block(candidate)) return; // for intrabc - int index = 0, ref; - assert(weight % 2 == 0); - - if (rf[1] == NONE_FRAME) { - // single reference frame - for (ref = 0; ref < 2; ++ref) { - if (candidate->ref_frame[ref] == rf[0]) { - int_mv this_refmv; - if (is_global_mv_block(candidate, gm_params[rf[0]].wmtype)) - this_refmv = gm_mv_candidates[0]; - else - this_refmv = get_sub_block_mv(candidate, ref, col); - - for (index = 0; index < *refmv_count; ++index) - if (ref_mv_stack[index].this_mv.as_int == this_refmv.as_int) break; - - if (index < *refmv_count) ref_mv_stack[index].weight += weight; - - // Add a new item to the list. - if (index == *refmv_count && *refmv_count < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[index].this_mv = this_refmv; - ref_mv_stack[index].weight = weight; - ++(*refmv_count); - } - if (have_newmv_in_inter_mode(candidate->mode)) ++*newmv_count; - ++*ref_match_count; - } - } - } else { - // compound reference frame - if (candidate->ref_frame[0] == rf[0] && candidate->ref_frame[1] == rf[1]) { - int_mv this_refmv[2]; - - for (ref = 0; ref < 2; ++ref) { - if (is_global_mv_block(candidate, gm_params[rf[ref]].wmtype)) - this_refmv[ref] = gm_mv_candidates[ref]; - else - this_refmv[ref] = get_sub_block_mv(candidate, ref, col); - } - - for (index = 0; index < *refmv_count; ++index) - if ((ref_mv_stack[index].this_mv.as_int == this_refmv[0].as_int) && - (ref_mv_stack[index].comp_mv.as_int == this_refmv[1].as_int)) - break; - - if (index < *refmv_count) ref_mv_stack[index].weight += weight; - - // Add a new item to the list. - if (index == *refmv_count && *refmv_count < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[index].this_mv = this_refmv[0]; - ref_mv_stack[index].comp_mv = this_refmv[1]; - ref_mv_stack[index].weight = weight; - ++(*refmv_count); - } - if (have_newmv_in_inter_mode(candidate->mode)) ++*newmv_count; - ++*ref_match_count; - } - } -} - -static void scan_row_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, - const MV_REFERENCE_FRAME rf[2], int row_offset, - CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, int max_row_offset, - int *processed_rows) { - int end_mi = AOMMIN(xd->n8_w, cm->mi_cols - mi_col); - end_mi = AOMMIN(end_mi, mi_size_wide[BLOCK_64X64]); - const int n8_w_8 = mi_size_wide[BLOCK_8X8]; - const int n8_w_16 = mi_size_wide[BLOCK_16X16]; - int i; - int col_offset = 0; - const int shift = 0; - // TODO(jingning): Revisit this part after cb4x4 is stable. - if (abs(row_offset) > 1) { - col_offset = 1; - if ((mi_col & 0x01) && xd->n8_w < n8_w_8) --col_offset; - } - const int use_step_16 = (xd->n8_w >= 16); - MB_MODE_INFO *const candidate_mi0 = xd->mi + row_offset * xd->mi_stride; - (void)mi_row; - - for (i = 0; i < end_mi;) { - const MB_MODE_INFO *const candidate = &candidate_mi0[col_offset + i]; - const int candidate_bsize = candidate->sb_type; - const int n8_w = mi_size_wide[candidate_bsize]; - int len = AOMMIN(xd->n8_w, n8_w); - if (use_step_16) - len = AOMMAX(n8_w_16, len); - else if (abs(row_offset) > 1) - len = AOMMAX(len, n8_w_8); - - int weight = 2; - if (xd->n8_w >= n8_w_8 && xd->n8_w <= n8_w) { - int inc = AOMMIN(-max_row_offset + row_offset + 1, - mi_size_high[candidate_bsize]); - // Obtain range used in weight calculation. - weight = AOMMAX(weight, (inc << shift)); - // Update processed rows. - *processed_rows = inc - row_offset - 1; - } - - add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, col_offset + i, len * weight); - - i += len; - } -} - -static void scan_col_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, - const MV_REFERENCE_FRAME rf[2], int col_offset, - CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, int max_col_offset, - int *processed_cols) { - int end_mi = AOMMIN(xd->n8_h, cm->mi_rows - mi_row); - end_mi = AOMMIN(end_mi, mi_size_high[BLOCK_64X64]); - const int n8_h_8 = mi_size_high[BLOCK_8X8]; - const int n8_h_16 = mi_size_high[BLOCK_16X16]; - int i; - int row_offset = 0; - const int shift = 0; - if (abs(col_offset) > 1) { - row_offset = 1; - if ((mi_row & 0x01) && xd->n8_h < n8_h_8) --row_offset; - } - const int use_step_16 = (xd->n8_h >= 16); - (void)mi_col; - - for (i = 0; i < end_mi;) { - const MB_MODE_INFO *const candidate = - &xd->mi[(row_offset + i) * xd->mi_stride + col_offset]; - const int candidate_bsize = candidate->sb_type; - const int n8_h = mi_size_high[candidate_bsize]; - int len = AOMMIN(xd->n8_h, n8_h); - if (use_step_16) - len = AOMMAX(n8_h_16, len); - else if (abs(col_offset) > 1) - len = AOMMAX(len, n8_h_8); - - int weight = 2; - if (xd->n8_h >= n8_h_8 && xd->n8_h <= n8_h) { - int inc = AOMMIN(-max_col_offset + col_offset + 1, - mi_size_wide[candidate_bsize]); - // Obtain range used in weight calculation. - weight = AOMMAX(weight, (inc << shift)); - // Update processed cols. - *processed_cols = inc - col_offset - 1; - } - - add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, col_offset, len * weight); - - i += len; - } -} - -static void scan_blk_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, - const int mi_row, const int mi_col, - const MV_REFERENCE_FRAME rf[2], int row_offset, - int col_offset, CANDIDATE_MV *ref_mv_stack, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, - uint8_t refmv_count[MODE_CTX_REF_FRAMES]) { - const TileInfo *const tile = &xd->tile; - POSITION mi_pos; - - mi_pos.row = row_offset; - mi_pos.col = col_offset; - - if (is_inside(tile, mi_col, mi_row, cm->mi_rows, &mi_pos)) { - const MB_MODE_INFO *const candidate = - &xd->mi[mi_pos.row * xd->mi_stride + mi_pos.col]; - const int len = mi_size_wide[BLOCK_8X8]; - - add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, mi_pos.col, 2 * len); - } // Analyze a single 8x8 block motion information. -} - -static int has_top_right(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, int bs) { - const int sb_mi_size = mi_size_wide[cm->seq_params.sb_size]; - const int mask_row = mi_row & (sb_mi_size - 1); - const int mask_col = mi_col & (sb_mi_size - 1); - - if (bs > mi_size_wide[BLOCK_64X64]) return 0; - - // In a split partition all apart from the bottom right has a top right - int has_tr = !((mask_row & bs) && (mask_col & bs)); - - // bs > 0 and bs is a power of 2 - assert(bs > 0 && !(bs & (bs - 1))); - - // For each 4x4 group of blocks, when the bottom right is decoded the blocks - // to the right have not been decoded therefore the bottom right does - // not have a top right - while (bs < sb_mi_size) { - if (mask_col & bs) { - if ((mask_col & (2 * bs)) && (mask_row & (2 * bs))) { - has_tr = 0; - break; - } - } else { - break; - } - bs <<= 1; - } - - // The left hand of two vertical rectangles always has a top right (as the - // block above will have been decoded) - if (xd->n8_w < xd->n8_h) - if (!xd->is_sec_rect) has_tr = 1; - - // The bottom of two horizontal rectangles never has a top right (as the block - // to the right won't have been decoded) - if (xd->n8_w > xd->n8_h) - if (xd->is_sec_rect) has_tr = 0; - - // The bottom left square of a Vertical A (in the old format) does - // not have a top right as it is decoded before the right hand - // rectangle of the partition - if (xd->cur_mi.partition == PARTITION_VERT_A) { - if (xd->n8_w == xd->n8_h) - if (mask_row & bs) has_tr = 0; - } - - return has_tr; -} - -static int check_sb_border(const int mi_row, const int mi_col, - const int row_offset, const int col_offset) { - const int sb_mi_size = mi_size_wide[BLOCK_64X64]; - const int row = mi_row & (sb_mi_size - 1); - const int col = mi_col & (sb_mi_size - 1); - - if (row + row_offset < 0 || row + row_offset >= sb_mi_size || - col + col_offset < 0 || col + col_offset >= sb_mi_size) - return 0; - - return 1; -} - -static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, MV_REFERENCE_FRAME ref_frame, - int blk_row, int blk_col, int_mv *gm_mv_candidates, - uint8_t refmv_count[MODE_CTX_REF_FRAMES], - CANDIDATE_MV ref_mv_stacks[][MAX_REF_MV_STACK_SIZE], - int16_t *mode_context) { - POSITION mi_pos; - int idx; - const int weight_unit = 1; // mi_size_wide[BLOCK_8X8]; - - mi_pos.row = (mi_row & 0x01) ? blk_row : blk_row + 1; - mi_pos.col = (mi_col & 0x01) ? blk_col : blk_col + 1; - - if (!is_inside(&xd->tile, mi_col, mi_row, cm->mi_rows, &mi_pos)) return 0; - - const TPL_MV_REF *prev_frame_mvs = - cm->tpl_mvs + ((mi_row + mi_pos.row) >> 1) * (cm->mi_stride >> 1) + - ((mi_col + mi_pos.col) >> 1); - - MV_REFERENCE_FRAME rf[2]; - av1_set_ref_frame(rf, ref_frame); - - if (rf[1] == NONE_FRAME) { - int cur_frame_index = cm->cur_frame.cur_frame_offset; - int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx; - int frame0_index = cm->buffer_pool.frame_bufs[buf_idx_0].cur_frame_offset; - int cur_offset_0 = get_relative_dist(cm, cur_frame_index, frame0_index); - CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[rf[0]]; - - if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { - int_mv this_refmv; - - get_mv_projection(&this_refmv.as_mv, prev_frame_mvs->mfmv0.as_mv, - cur_offset_0, prev_frame_mvs->ref_frame_offset); - lower_mv_precision(&this_refmv.as_mv, cm->allow_high_precision_mv, - cm->cur_frame_force_integer_mv); - - if (blk_row == 0 && blk_col == 0) - if (abs(this_refmv.as_mv.row - gm_mv_candidates[0].as_mv.row) >= 16 || - abs(this_refmv.as_mv.col - gm_mv_candidates[0].as_mv.col) >= 16) - mode_context[ref_frame] |= (1 << GLOBALMV_OFFSET); - - for (idx = 0; idx < refmv_count[rf[0]]; ++idx) - if (this_refmv.as_int == ref_mv_stack[idx].this_mv.as_int) break; - - if (idx < refmv_count[rf[0]]) ref_mv_stack[idx].weight += 2 * weight_unit; - - if (idx == refmv_count[rf[0]] && - refmv_count[rf[0]] < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[idx].this_mv.as_int = this_refmv.as_int; - ref_mv_stack[idx].weight = 2 * weight_unit; - ++(refmv_count[rf[0]]); - } - - return 1; - } - } else { - // Process compound inter mode - int cur_frame_index = cm->cur_frame.cur_frame_offset; - int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx; - int frame0_index = cm->buffer_pool.frame_bufs[buf_idx_0].cur_frame_offset; - - int cur_offset_0 = get_relative_dist(cm, cur_frame_index, frame0_index); - int buf_idx_1 = cm->frame_refs[FWD_RF_OFFSET(rf[1])].idx; - int frame1_index = cm->buffer_pool.frame_bufs[buf_idx_1].cur_frame_offset; - int cur_offset_1 = get_relative_dist(cm, cur_frame_index, frame1_index); - CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[ref_frame]; - - if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { - int_mv this_refmv; - int_mv comp_refmv; - get_mv_projection(&this_refmv.as_mv, prev_frame_mvs->mfmv0.as_mv, - cur_offset_0, prev_frame_mvs->ref_frame_offset); - get_mv_projection(&comp_refmv.as_mv, prev_frame_mvs->mfmv0.as_mv, - cur_offset_1, prev_frame_mvs->ref_frame_offset); - - lower_mv_precision(&this_refmv.as_mv, cm->allow_high_precision_mv, - cm->cur_frame_force_integer_mv); - lower_mv_precision(&comp_refmv.as_mv, cm->allow_high_precision_mv, - cm->cur_frame_force_integer_mv); - - if (blk_row == 0 && blk_col == 0) - if (abs(this_refmv.as_mv.row - gm_mv_candidates[0].as_mv.row) >= 16 || - abs(this_refmv.as_mv.col - gm_mv_candidates[0].as_mv.col) >= 16 || - abs(comp_refmv.as_mv.row - gm_mv_candidates[1].as_mv.row) >= 16 || - abs(comp_refmv.as_mv.col - gm_mv_candidates[1].as_mv.col) >= 16) - mode_context[ref_frame] |= (1 << GLOBALMV_OFFSET); - - for (idx = 0; idx < refmv_count[ref_frame]; ++idx) - if (this_refmv.as_int == ref_mv_stack[idx].this_mv.as_int && - comp_refmv.as_int == ref_mv_stack[idx].comp_mv.as_int) - break; - - if (idx < refmv_count[ref_frame]) - ref_mv_stack[idx].weight += 2 * weight_unit; - - if (idx == refmv_count[ref_frame] && - refmv_count[ref_frame] < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[idx].this_mv.as_int = this_refmv.as_int; - ref_mv_stack[idx].comp_mv.as_int = comp_refmv.as_int; - ref_mv_stack[idx].weight = 2 * weight_unit; - ++(refmv_count[ref_frame]); - } - return 1; - } - } - return 0; -} - -static void setup_ref_mv_list( - const AV1_COMMON *cm, const MACROBLOCKD *xd, MV_REFERENCE_FRAME ref_frame, - uint8_t refmv_count[MODE_CTX_REF_FRAMES], - CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], - int_mv mv_ref_list[][MAX_MV_REF_CANDIDATES], int_mv *gm_mv_candidates, - int mi_row, int mi_col, int16_t *mode_context) { - const int bs = AOMMAX(xd->n8_w, xd->n8_h); - const int has_tr = has_top_right(cm, xd, mi_row, mi_col, bs); - MV_REFERENCE_FRAME rf[2]; - - const TileInfo *const tile = &xd->tile; - int max_row_offset = 0, max_col_offset = 0; - const int row_adj = (xd->n8_h < mi_size_high[BLOCK_8X8]) && (mi_row & 0x01); - const int col_adj = (xd->n8_w < mi_size_wide[BLOCK_8X8]) && (mi_col & 0x01); - int processed_rows = 0; - int processed_cols = 0; - - av1_set_ref_frame(rf, ref_frame); - mode_context[ref_frame] = 0; - refmv_count[ref_frame] = 0; - - // Find valid maximum row/col offset. - if (xd->up_available) { - max_row_offset = -(MVREF_ROW_COLS << 1) + row_adj; - - if (xd->n8_h < mi_size_high[BLOCK_8X8]) - max_row_offset = -(2 << 1) + row_adj; - - max_row_offset = - find_valid_row_offset(tile, mi_row, cm->mi_rows, max_row_offset); - } - - if (xd->left_available) { - max_col_offset = -(MVREF_ROW_COLS << 1) + col_adj; - - if (xd->n8_w < mi_size_wide[BLOCK_8X8]) - max_col_offset = -(2 << 1) + col_adj; - - max_col_offset = find_valid_col_offset(tile, mi_col, max_col_offset); - } - - uint8_t col_match_count = 0; - uint8_t row_match_count = 0; - uint8_t newmv_count = 0; - - // Scan the first above row mode info. row_offset = -1; - if (abs(max_row_offset) >= 1) - scan_row_mbmi(cm, xd, mi_row, mi_col, rf, -1, ref_mv_stack[ref_frame], - &refmv_count[ref_frame], &row_match_count, &newmv_count, - gm_mv_candidates, max_row_offset, &processed_rows); - // Scan the first left column mode info. col_offset = -1; - if (abs(max_col_offset) >= 1) - scan_col_mbmi(cm, xd, mi_row, mi_col, rf, -1, ref_mv_stack[ref_frame], - &refmv_count[ref_frame], &col_match_count, &newmv_count, - gm_mv_candidates, max_col_offset, &processed_cols); - // Check top-right boundary - if (has_tr) - scan_blk_mbmi(cm, xd, mi_row, mi_col, rf, -1, xd->n8_w, - ref_mv_stack[ref_frame], &row_match_count, &newmv_count, - gm_mv_candidates, &refmv_count[ref_frame]); - - uint8_t nearest_match = (row_match_count > 0) + (col_match_count > 0); - uint8_t nearest_refmv_count = refmv_count[ref_frame]; - - // TODO(yunqing): for comp_search, do it for all 3 cases. - for (int idx = 0; idx < nearest_refmv_count; ++idx) - ref_mv_stack[ref_frame][idx].weight += REF_CAT_LEVEL; - - if (cm->allow_ref_frame_mvs) { - int is_available = 0; - const int voffset = AOMMAX(mi_size_high[BLOCK_8X8], xd->n8_h); - const int hoffset = AOMMAX(mi_size_wide[BLOCK_8X8], xd->n8_w); - const int blk_row_end = AOMMIN(xd->n8_h, mi_size_high[BLOCK_64X64]); - const int blk_col_end = AOMMIN(xd->n8_w, mi_size_wide[BLOCK_64X64]); - - const int tpl_sample_pos[3][2] = { - { voffset, -2 }, - { voffset, hoffset }, - { voffset - 2, hoffset }, - }; - const int allow_extension = (xd->n8_h >= mi_size_high[BLOCK_8X8]) && - (xd->n8_h < mi_size_high[BLOCK_64X64]) && - (xd->n8_w >= mi_size_wide[BLOCK_8X8]) && - (xd->n8_w < mi_size_wide[BLOCK_64X64]); - - int step_h = (xd->n8_h >= mi_size_high[BLOCK_64X64]) - ? mi_size_high[BLOCK_16X16] - : mi_size_high[BLOCK_8X8]; - int step_w = (xd->n8_w >= mi_size_wide[BLOCK_64X64]) - ? mi_size_wide[BLOCK_16X16] - : mi_size_wide[BLOCK_8X8]; - - for (int blk_row = 0; blk_row < blk_row_end; blk_row += step_h) { - for (int blk_col = 0; blk_col < blk_col_end; blk_col += step_w) { - int ret = add_tpl_ref_mv(cm, xd, mi_row, mi_col, ref_frame, blk_row, - blk_col, gm_mv_candidates, refmv_count, - ref_mv_stack, mode_context); - if (blk_row == 0 && blk_col == 0) is_available = ret; - } - } - - if (is_available == 0) mode_context[ref_frame] |= (1 << GLOBALMV_OFFSET); - - for (int i = 0; i < 3 && allow_extension; ++i) { - const int blk_row = tpl_sample_pos[i][0]; - const int blk_col = tpl_sample_pos[i][1]; - - if (!check_sb_border(mi_row, mi_col, blk_row, blk_col)) continue; - add_tpl_ref_mv(cm, xd, mi_row, mi_col, ref_frame, blk_row, blk_col, - gm_mv_candidates, refmv_count, ref_mv_stack, mode_context); - } - } - - uint8_t dummy_newmv_count = 0; - - // Scan the second outer area. - scan_blk_mbmi(cm, xd, mi_row, mi_col, rf, -1, -1, ref_mv_stack[ref_frame], - &row_match_count, &dummy_newmv_count, gm_mv_candidates, - &refmv_count[ref_frame]); - - for (int idx = 2; idx <= MVREF_ROW_COLS; ++idx) { - const int row_offset = -(idx << 1) + 1 + row_adj; - const int col_offset = -(idx << 1) + 1 + col_adj; - - if (abs(row_offset) <= abs(max_row_offset) && - abs(row_offset) > processed_rows) - scan_row_mbmi(cm, xd, mi_row, mi_col, rf, row_offset, - ref_mv_stack[ref_frame], &refmv_count[ref_frame], - &row_match_count, &dummy_newmv_count, gm_mv_candidates, - max_row_offset, &processed_rows); - - if (abs(col_offset) <= abs(max_col_offset) && - abs(col_offset) > processed_cols) - scan_col_mbmi(cm, xd, mi_row, mi_col, rf, col_offset, - ref_mv_stack[ref_frame], &refmv_count[ref_frame], - &col_match_count, &dummy_newmv_count, gm_mv_candidates, - max_col_offset, &processed_cols); - } - - uint8_t ref_match_count = (row_match_count > 0) + (col_match_count > 0); - - switch (nearest_match) { - case 0: - mode_context[ref_frame] |= 0; - if (ref_match_count >= 1) mode_context[ref_frame] |= 1; - if (ref_match_count == 1) - mode_context[ref_frame] |= (1 << REFMV_OFFSET); - else if (ref_match_count >= 2) - mode_context[ref_frame] |= (2 << REFMV_OFFSET); - break; - case 1: - mode_context[ref_frame] |= (newmv_count > 0) ? 2 : 3; - if (ref_match_count == 1) - mode_context[ref_frame] |= (3 << REFMV_OFFSET); - else if (ref_match_count >= 2) - mode_context[ref_frame] |= (4 << REFMV_OFFSET); - break; - case 2: - default: - if (newmv_count >= 1) - mode_context[ref_frame] |= 4; - else - mode_context[ref_frame] |= 5; - - mode_context[ref_frame] |= (5 << REFMV_OFFSET); - break; - } - - // Rank the likelihood and assign nearest and near mvs. - int len = nearest_refmv_count; - while (len > 0) { - int nr_len = 0; - for (int idx = 1; idx < len; ++idx) { - if (ref_mv_stack[ref_frame][idx - 1].weight < - ref_mv_stack[ref_frame][idx].weight) { - CANDIDATE_MV tmp_mv = ref_mv_stack[ref_frame][idx - 1]; - ref_mv_stack[ref_frame][idx - 1] = ref_mv_stack[ref_frame][idx]; - ref_mv_stack[ref_frame][idx] = tmp_mv; - nr_len = idx; - } - } - len = nr_len; - } - - len = refmv_count[ref_frame]; - while (len > nearest_refmv_count) { - int nr_len = nearest_refmv_count; - for (int idx = nearest_refmv_count + 1; idx < len; ++idx) { - if (ref_mv_stack[ref_frame][idx - 1].weight < - ref_mv_stack[ref_frame][idx].weight) { - CANDIDATE_MV tmp_mv = ref_mv_stack[ref_frame][idx - 1]; - ref_mv_stack[ref_frame][idx - 1] = ref_mv_stack[ref_frame][idx]; - ref_mv_stack[ref_frame][idx] = tmp_mv; - nr_len = idx; - } - } - len = nr_len; - } - - if (rf[1] > NONE_FRAME) { - // TODO(jingning, yunqing): Refactor and consolidate the compound and - // single reference frame modes. Reduce unnecessary redundancy. - if (refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES) { - int_mv ref_id[2][2], ref_diff[2][2]; - int ref_id_count[2] = { 0 }, ref_diff_count[2] = { 0 }; - - int mi_width = AOMMIN(mi_size_wide[BLOCK_64X64], xd->n8_w); - mi_width = AOMMIN(mi_width, cm->mi_cols - mi_col); - int mi_height = AOMMIN(mi_size_high[BLOCK_64X64], xd->n8_h); - mi_height = AOMMIN(mi_height, cm->mi_rows - mi_row); - int mi_size = AOMMIN(mi_width, mi_height); - - for (int idx = 0; abs(max_row_offset) >= 1 && idx < mi_size;) { - const MB_MODE_INFO *const candidate = &xd->mi[-xd->mi_stride + idx]; - const int candidate_bsize = candidate->sb_type; - - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - MV_REFERENCE_FRAME can_rf = candidate->ref_frame[rf_idx]; - - for (int cmp_idx = 0; cmp_idx < 2; ++cmp_idx) { - if (can_rf == rf[cmp_idx] && ref_id_count[cmp_idx] < 2) { - ref_id[cmp_idx][ref_id_count[cmp_idx]] = candidate->mv[rf_idx]; - ++ref_id_count[cmp_idx]; - } else if (can_rf > INTRA_FRAME && ref_diff_count[cmp_idx] < 2) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[can_rf] != - cm->ref_frame_sign_bias[rf[cmp_idx]]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - ref_diff[cmp_idx][ref_diff_count[cmp_idx]] = this_mv; - ++ref_diff_count[cmp_idx]; - } - } - } - idx += mi_size_wide[candidate_bsize]; - } - - for (int idx = 0; abs(max_col_offset) >= 1 && idx < mi_size;) { - const MB_MODE_INFO *const candidate = &xd->mi[idx * xd->mi_stride - 1]; - const int candidate_bsize = candidate->sb_type; - - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - MV_REFERENCE_FRAME can_rf = candidate->ref_frame[rf_idx]; - - for (int cmp_idx = 0; cmp_idx < 2; ++cmp_idx) { - if (can_rf == rf[cmp_idx] && ref_id_count[cmp_idx] < 2) { - ref_id[cmp_idx][ref_id_count[cmp_idx]] = candidate->mv[rf_idx]; - ++ref_id_count[cmp_idx]; - } else if (can_rf > INTRA_FRAME && ref_diff_count[cmp_idx] < 2) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[can_rf] != - cm->ref_frame_sign_bias[rf[cmp_idx]]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - ref_diff[cmp_idx][ref_diff_count[cmp_idx]] = this_mv; - ++ref_diff_count[cmp_idx]; - } - } - } - idx += mi_size_high[candidate_bsize]; - } - - // Build up the compound mv predictor - int_mv comp_list[3][2]; - - for (int idx = 0; idx < 2; ++idx) { - int comp_idx = 0; - for (int list_idx = 0; list_idx < ref_id_count[idx] && comp_idx < 2; - ++list_idx, ++comp_idx) - comp_list[comp_idx][idx] = ref_id[idx][list_idx]; - for (int list_idx = 0; list_idx < ref_diff_count[idx] && comp_idx < 2; - ++list_idx, ++comp_idx) - comp_list[comp_idx][idx] = ref_diff[idx][list_idx]; - for (; comp_idx < 3; ++comp_idx) - comp_list[comp_idx][idx] = gm_mv_candidates[idx]; - } - - if (refmv_count[ref_frame]) { - assert(refmv_count[ref_frame] == 1); - if (comp_list[0][0].as_int == - ref_mv_stack[ref_frame][0].this_mv.as_int && - comp_list[0][1].as_int == - ref_mv_stack[ref_frame][0].comp_mv.as_int) { - ref_mv_stack[ref_frame][refmv_count[ref_frame]].this_mv = - comp_list[1][0]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = - comp_list[1][1]; - } else { - ref_mv_stack[ref_frame][refmv_count[ref_frame]].this_mv = - comp_list[0][0]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = - comp_list[0][1]; - } - ref_mv_stack[ref_frame][refmv_count[ref_frame]].weight = 2; - ++refmv_count[ref_frame]; - } else { - for (int idx = 0; idx < MAX_MV_REF_CANDIDATES; ++idx) { - ref_mv_stack[ref_frame][refmv_count[ref_frame]].this_mv = - comp_list[idx][0]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = - comp_list[idx][1]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].weight = 2; - ++refmv_count[ref_frame]; - } - } - } - - assert(refmv_count[ref_frame] >= 2); - - for (int idx = 0; idx < refmv_count[ref_frame]; ++idx) { - clamp_mv_ref(&ref_mv_stack[ref_frame][idx].this_mv.as_mv, - xd->n8_w << MI_SIZE_LOG2, xd->n8_h << MI_SIZE_LOG2, xd); - clamp_mv_ref(&ref_mv_stack[ref_frame][idx].comp_mv.as_mv, - xd->n8_w << MI_SIZE_LOG2, xd->n8_h << MI_SIZE_LOG2, xd); - } - } else { - // Handle single reference frame extension - int mi_width = AOMMIN(mi_size_wide[BLOCK_64X64], xd->n8_w); - mi_width = AOMMIN(mi_width, cm->mi_cols - mi_col); - int mi_height = AOMMIN(mi_size_high[BLOCK_64X64], xd->n8_h); - mi_height = AOMMIN(mi_height, cm->mi_rows - mi_row); - int mi_size = AOMMIN(mi_width, mi_height); - - for (int idx = 0; abs(max_row_offset) >= 1 && idx < mi_size && - refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES;) { - const MB_MODE_INFO *const candidate = &xd->mi[-xd->mi_stride + idx]; - const int candidate_bsize = candidate->sb_type; - - // TODO(jingning): Refactor the following code. - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - if (candidate->ref_frame[rf_idx] > INTRA_FRAME) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[candidate->ref_frame[rf_idx]] != - cm->ref_frame_sign_bias[ref_frame]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - int stack_idx; - for (stack_idx = 0; stack_idx < refmv_count[ref_frame]; ++stack_idx) { - int_mv stack_mv = ref_mv_stack[ref_frame][stack_idx].this_mv; - if (this_mv.as_int == stack_mv.as_int) break; - } - - if (stack_idx == refmv_count[ref_frame]) { - ref_mv_stack[ref_frame][stack_idx].this_mv = this_mv; - - // TODO(jingning): Set an arbitrary small number here. The weight - // doesn't matter as long as it is properly initialized. - ref_mv_stack[ref_frame][stack_idx].weight = 2; - ++refmv_count[ref_frame]; - } - } - } - idx += mi_size_wide[candidate_bsize]; - } - - for (int idx = 0; abs(max_col_offset) >= 1 && idx < mi_size && - refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES;) { - const MB_MODE_INFO *const candidate = &xd->mi[idx * xd->mi_stride - 1]; - const int candidate_bsize = candidate->sb_type; - - // TODO(jingning): Refactor the following code. - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - if (candidate->ref_frame[rf_idx] > INTRA_FRAME) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[candidate->ref_frame[rf_idx]] != - cm->ref_frame_sign_bias[ref_frame]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - int stack_idx; - for (stack_idx = 0; stack_idx < refmv_count[ref_frame]; ++stack_idx) { - int_mv stack_mv = ref_mv_stack[ref_frame][stack_idx].this_mv; - if (this_mv.as_int == stack_mv.as_int) break; - } - - if (stack_idx == refmv_count[ref_frame]) { - ref_mv_stack[ref_frame][stack_idx].this_mv = this_mv; - - // TODO(jingning): Set an arbitrary small number here. The weight - // doesn't matter as long as it is properly initialized. - ref_mv_stack[ref_frame][stack_idx].weight = 2; - ++refmv_count[ref_frame]; - } - } - } - idx += mi_size_high[candidate_bsize]; - } - - for (int idx = 0; idx < refmv_count[ref_frame]; ++idx) { - clamp_mv_ref(&ref_mv_stack[ref_frame][idx].this_mv.as_mv, - xd->n8_w << MI_SIZE_LOG2, xd->n8_h << MI_SIZE_LOG2, xd); - } - - if (mv_ref_list != NULL) { - for (int idx = refmv_count[ref_frame]; idx < MAX_MV_REF_CANDIDATES; ++idx) - mv_ref_list[rf[0]][idx].as_int = gm_mv_candidates[0].as_int; - - for (int idx = 0; - idx < AOMMIN(MAX_MV_REF_CANDIDATES, refmv_count[ref_frame]); ++idx) { - mv_ref_list[rf[0]][idx].as_int = - ref_mv_stack[ref_frame][idx].this_mv.as_int; - } - } - } -} - -static void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd, - MB_MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, - uint8_t ref_mv_count[MODE_CTX_REF_FRAMES], - CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], - int_mv mv_ref_list[][MAX_MV_REF_CANDIDATES], - int_mv *global_mvs, int mi_row, int mi_col, - int16_t *mode_context) { - int_mv zeromv[2]; - BLOCK_SIZE bsize = mi->sb_type; - MV_REFERENCE_FRAME rf[2]; - av1_set_ref_frame(rf, ref_frame); - - if (ref_frame < REF_FRAMES) { - if (ref_frame != INTRA_FRAME) { - global_mvs[ref_frame] = gm_get_motion_vector( - &cm->global_motion[ref_frame], cm->allow_high_precision_mv, bsize, - mi_col, mi_row, cm->cur_frame_force_integer_mv); - } else { - global_mvs[ref_frame].as_int = INVALID_MV; - } - } - - if (ref_frame != INTRA_FRAME) { - zeromv[0].as_int = - gm_get_motion_vector(&cm->global_motion[rf[0]], - cm->allow_high_precision_mv, bsize, mi_col, mi_row, - cm->cur_frame_force_integer_mv) - .as_int; - zeromv[1].as_int = - (rf[1] != NONE_FRAME) - ? gm_get_motion_vector(&cm->global_motion[rf[1]], - cm->allow_high_precision_mv, bsize, mi_col, - mi_row, cm->cur_frame_force_integer_mv) - .as_int - : 0; - } else { - zeromv[0].as_int = zeromv[1].as_int = 0; - } - - setup_ref_mv_list(cm, xd, ref_frame, ref_mv_count, ref_mv_stack, mv_ref_list, - zeromv, mi_row, mi_col, mode_context); -} - -static void av1_setup_frame_buf_refs(AV1_COMMON *cm) { - cm->cur_frame.cur_frame_offset = cm->frame_offset; - - MV_REFERENCE_FRAME ref_frame; - for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx; - if (buf_idx >= 0) - cm->cur_frame.ref_frame_offset[ref_frame - LAST_FRAME] = - cm->buffer_pool.frame_bufs[buf_idx].cur_frame_offset; - } -} - -#define MAX_OFFSET_WIDTH 64 -#define MAX_OFFSET_HEIGHT 0 - -static int get_block_position(AV1_COMMON *cm, int *mi_r, int *mi_c, int blk_row, - int blk_col, MV mv, int sign_bias) { - const int base_blk_row = (blk_row >> 3) << 3; - const int base_blk_col = (blk_col >> 3) << 3; - - const int row_offset = (mv.row >= 0) ? (mv.row >> (4 + MI_SIZE_LOG2)) - : -((-mv.row) >> (4 + MI_SIZE_LOG2)); - - const int col_offset = (mv.col >= 0) ? (mv.col >> (4 + MI_SIZE_LOG2)) - : -((-mv.col) >> (4 + MI_SIZE_LOG2)); - - int row = (sign_bias == 1) ? blk_row - row_offset : blk_row + row_offset; - int col = (sign_bias == 1) ? blk_col - col_offset : blk_col + col_offset; - - if (row < 0 || row >= (cm->mi_rows >> 1) || col < 0 || - col >= (cm->mi_cols >> 1)) - return 0; - - if (row < base_blk_row - (MAX_OFFSET_HEIGHT >> 3) || - row >= base_blk_row + 8 + (MAX_OFFSET_HEIGHT >> 3) || - col < base_blk_col - (MAX_OFFSET_WIDTH >> 3) || - col >= base_blk_col + 8 + (MAX_OFFSET_WIDTH >> 3)) - return 0; - - *mi_r = row; - *mi_c = col; - - return 1; -} - -static int motion_field_projection(AV1_COMMON *cm, MV_REFERENCE_FRAME ref_frame, - int dir, - const int from_x4, const int to_x4, - const int from_y4, const int to_y4) { - TPL_MV_REF *tpl_mvs_base = cm->tpl_mvs; - int ref_offset[TOTAL_REFS_PER_FRAME] = { 0 }; - int ref_sign[TOTAL_REFS_PER_FRAME] = { 0 }; - - (void)dir; - - int ref_frame_idx = cm->frame_refs[FWD_RF_OFFSET(ref_frame)].idx; - if (ref_frame_idx < 0) return 0; - - if (cm->buffer_pool.frame_bufs[ref_frame_idx].intra_only) return 0; - - if (cm->buffer_pool.frame_bufs[ref_frame_idx].mi_rows != cm->mi_rows || - cm->buffer_pool.frame_bufs[ref_frame_idx].mi_cols != cm->mi_cols) - return 0; - - int ref_frame_index = - cm->buffer_pool.frame_bufs[ref_frame_idx].cur_frame_offset; - unsigned int *ref_rf_idx = - &cm->buffer_pool.frame_bufs[ref_frame_idx].ref_frame_offset[0]; - int cur_frame_index = cm->cur_frame.cur_frame_offset; - int ref_to_cur = get_relative_dist(cm, ref_frame_index, cur_frame_index); - - for (MV_REFERENCE_FRAME rf = LAST_FRAME; rf <= INTER_REFS_PER_FRAME; ++rf) { - ref_offset[rf] = - get_relative_dist(cm, ref_frame_index, ref_rf_idx[rf - LAST_FRAME]); - // note the inverted sign - ref_sign[rf] = - get_relative_dist(cm, ref_rf_idx[rf - LAST_FRAME], ref_frame_index) < 0; - } - - if (dir == 2) ref_to_cur = -ref_to_cur; - - MV_REF *mv_ref_base = cm->buffer_pool.frame_bufs[ref_frame_idx].mvs; - const ptrdiff_t mv_stride = - cm->buffer_pool.frame_bufs[ref_frame_idx].mv_stride; - const int mvs_rows = (cm->mi_rows + 1) >> 1; - const int mvs_cols = (cm->mi_cols + 1) >> 1; - - assert(from_y4 >= 0); - const int row_start8 = from_y4 >> 1; - const int row_end8 = imin(to_y4 >> 1, mvs_rows); - const int col_start8 = imax((from_x4 - (MAX_OFFSET_WIDTH >> 2)) >> 1, 0); - const int col_end8 = imin((to_x4 + (MAX_OFFSET_WIDTH >> 2)) >> 1, mvs_cols); - for (int blk_row = row_start8; blk_row < row_end8; ++blk_row) { - for (int blk_col = col_start8; blk_col < col_end8; ++blk_col) { - MV_REF *mv_ref = &mv_ref_base[((blk_row << 1) + 1) * mv_stride + - (blk_col << 1) + 1]; - int diridx; - const int ref0 = mv_ref->ref_frame[0], ref1 = mv_ref->ref_frame[1]; - if (ref1 > 0 && ref_sign[ref1] && - abs(mv_ref->mv[1].as_mv.row) < (1 << 12) && - abs(mv_ref->mv[1].as_mv.col) < (1 << 12)) - { - diridx = 1; - } else if (ref0 > 0 && ref_sign[ref0] && - abs(mv_ref->mv[0].as_mv.row) < (1 << 12) && - abs(mv_ref->mv[0].as_mv.col) < (1 << 12)) - { - diridx = 0; - } else { - continue; - } - MV fwd_mv = mv_ref->mv[diridx].as_mv; - - if (mv_ref->ref_frame[diridx] > INTRA_FRAME) { - int_mv this_mv; - int mi_r, mi_c; - const int ref_frame_offset = ref_offset[mv_ref->ref_frame[diridx]]; - - int pos_valid = abs(ref_frame_offset) <= MAX_FRAME_DISTANCE && - ref_frame_offset > 0 && - abs(ref_to_cur) <= MAX_FRAME_DISTANCE; - - if (pos_valid) { - get_mv_projection(&this_mv.as_mv, fwd_mv, ref_to_cur, - ref_frame_offset); - pos_valid = get_block_position(cm, &mi_r, &mi_c, blk_row, blk_col, - this_mv.as_mv, dir >> 1); - } - - if (pos_valid && mi_c >= (from_x4 >> 1) && mi_c < (to_x4 >> 1)) { - int mi_offset = mi_r * (cm->mi_stride >> 1) + mi_c; - - tpl_mvs_base[mi_offset].mfmv0.as_mv.row = fwd_mv.row; - tpl_mvs_base[mi_offset].mfmv0.as_mv.col = fwd_mv.col; - tpl_mvs_base[mi_offset].ref_frame_offset = ref_frame_offset; - } - } - } - } - - return 1; -} - -static void av1_setup_motion_field(AV1_COMMON *cm) { - if (!cm->seq_params.enable_order_hint) return; - - TPL_MV_REF *tpl_mvs_base = cm->tpl_mvs; - int size = (((cm->mi_rows + 31) & ~31) >> 1) * (cm->mi_stride >> 1); - for (int idx = 0; idx < size; ++idx) { - tpl_mvs_base[idx].mfmv0.as_int = INVALID_MV; - tpl_mvs_base[idx].ref_frame_offset = 0; - } - - memset(cm->ref_frame_side, 0, sizeof(cm->ref_frame_side)); - RefCntBuffer *const frame_bufs = cm->buffer_pool.frame_bufs; - - const int cur_order_hint = cm->cur_frame.cur_frame_offset; - int *const ref_buf_idx = cm->ref_buf_idx; - int *const ref_order_hint = cm->ref_order_hint; - - for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { - const int ref_idx = ref_frame - LAST_FRAME; - const int buf_idx = cm->frame_refs[ref_idx].idx; - int order_hint = 0; - - if (buf_idx >= 0) order_hint = frame_bufs[buf_idx].cur_frame_offset; - - ref_buf_idx[ref_idx] = buf_idx; - ref_order_hint[ref_idx] = order_hint; - - if (get_relative_dist(cm, order_hint, cur_order_hint) > 0) - cm->ref_frame_side[ref_frame] = 1; - else if (order_hint == cur_order_hint) - cm->ref_frame_side[ref_frame] = -1; - } -} - -enum BlockSize { - BS_128x128, - BS_128x64, - BS_64x128, - BS_64x64, - BS_64x32, - BS_64x16, - BS_32x64, - BS_32x32, - BS_32x16, - BS_32x8, - BS_16x64, - BS_16x32, - BS_16x16, - BS_16x8, - BS_16x4, - BS_8x32, - BS_8x16, - BS_8x8, - BS_8x4, - BS_4x16, - BS_4x8, - BS_4x4, - N_BS_SIZES, -}; -extern const uint8_t dav1d_block_dimensions[N_BS_SIZES][4]; -const uint8_t dav1d_bs_to_sbtype[N_BS_SIZES] = { - [BS_128x128] = BLOCK_128X128, - [BS_128x64] = BLOCK_128X64, - [BS_64x128] = BLOCK_64X128, - [BS_64x64] = BLOCK_64X64, - [BS_64x32] = BLOCK_64X32, - [BS_64x16] = BLOCK_64X16, - [BS_32x64] = BLOCK_32X64, - [BS_32x32] = BLOCK_32X32, - [BS_32x16] = BLOCK_32X16, - [BS_32x8] = BLOCK_32X8, - [BS_16x64] = BLOCK_16X64, - [BS_16x32] = BLOCK_16X32, - [BS_16x16] = BLOCK_16X16, - [BS_16x8] = BLOCK_16X8, - [BS_16x4] = BLOCK_16X4, - [BS_8x32] = BLOCK_8X32, - [BS_8x16] = BLOCK_8X16, - [BS_8x8] = BLOCK_8X8, - [BS_8x4] = BLOCK_8X4, - [BS_4x16] = BLOCK_4X16, - [BS_4x8] = BLOCK_4X8, - [BS_4x4] = BLOCK_4X4, -}; -const uint8_t dav1d_sbtype_to_bs[BLOCK_SIZES_ALL] = { - [BLOCK_128X128] = BS_128x128, - [BLOCK_128X64] = BS_128x64, - [BLOCK_64X128] = BS_64x128, - [BLOCK_64X64] = BS_64x64, - [BLOCK_64X32] = BS_64x32, - [BLOCK_64X16] = BS_64x16, - [BLOCK_32X64] = BS_32x64, - [BLOCK_32X32] = BS_32x32, - [BLOCK_32X16] = BS_32x16, - [BLOCK_32X8] = BS_32x8, - [BLOCK_16X64] = BS_16x64, - [BLOCK_16X32] = BS_16x32, - [BLOCK_16X16] = BS_16x16, - [BLOCK_16X8] = BS_16x8, - [BLOCK_16X4] = BS_16x4, - [BLOCK_8X32] = BS_8x32, - [BLOCK_8X16] = BS_8x16, - [BLOCK_8X8] = BS_8x8, - [BLOCK_8X4] = BS_8x4, - [BLOCK_4X16] = BS_4x16, - [BLOCK_4X8] = BS_4x8, - [BLOCK_4X4] = BS_4x4, -}; - -#include - -void dav1d_find_ref_mvs(CANDIDATE_MV *mvstack, int *cnt, int_mv (*mvlist)[2], - int *ctx, int refidx_dav1d[2], - int w4, int h4, int bs, int bp, int by4, int bx4, - int tile_col_start4, int tile_col_end4, - int tile_row_start4, int tile_row_end4, - AV1_COMMON *cm); -void dav1d_find_ref_mvs(CANDIDATE_MV *mvstack, int *cnt, int_mv (*mvlist)[2], - int *ctx, int refidx_dav1d[2], - int w4, int h4, int bs, int bp, int by4, int bx4, - int tile_col_start4, int tile_col_end4, - int tile_row_start4, int tile_row_end4, - AV1_COMMON *cm) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - const int bh4 = dav1d_block_dimensions[bs][1]; - int stride = (int) cm->cur_frame.mv_stride; - MACROBLOCKD xd = (MACROBLOCKD) { - .n8_w = bw4, - .n8_h = bh4, - .mi_stride = stride, - .up_available = by4 > tile_row_start4, - .left_available = bx4 > tile_col_start4, - .tile = { - .mi_col_end = AOMMIN(w4, tile_col_end4), - .mi_row_end = AOMMIN(h4, tile_row_end4), - .tg_horz_boundary = 0, - .mi_row_start = tile_row_start4, - .mi_col_start = tile_col_start4, - }, - .mi = (MB_MODE_INFO *) &cm->cur_frame.mvs[by4 * stride + bx4], - .mb_to_bottom_edge = (h4 - bh4 - by4) * 32, - .mb_to_left_edge = -bx4 * 32, - .mb_to_right_edge = (w4 - bw4 - bx4) * 32, - .mb_to_top_edge = -by4 * 32, - .is_sec_rect = 0, - .cur_mi = { - .partition = bp, - }, - }; - xd.mi->sb_type = dav1d_bs_to_sbtype[bs]; - if (xd.n8_w < xd.n8_h) { - // Only mark is_sec_rect as 1 for the last block. - // For PARTITION_VERT_4, it would be (0, 0, 0, 1); - // For other partitions, it would be (0, 1). - if (!((bx4 + xd.n8_w) & (xd.n8_h - 1))) xd.is_sec_rect = 1; - } - - if (xd.n8_w > xd.n8_h) - if (by4 & (xd.n8_w - 1)) xd.is_sec_rect = 1; - - MV_REFERENCE_FRAME rf[2] = { refidx_dav1d[0] + 1, refidx_dav1d[1] + 1 }; - const int refidx = av1_ref_frame_type(rf); - int16_t single_context[MODE_CTX_REF_FRAMES]; - uint8_t mv_cnt[MODE_CTX_REF_FRAMES]; - CANDIDATE_MV mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; - int_mv mv_list[MODE_CTX_REF_FRAMES][MAX_MV_REF_CANDIDATES]; - int_mv gmvs[MODE_CTX_REF_FRAMES]; - av1_find_mv_refs(cm, &xd, xd.mi, refidx, mv_cnt, - mv_stack, mv_list, gmvs, by4, bx4, - single_context); - for (int i = 0; i < mv_cnt[refidx]; i++) - mvstack[i] = mv_stack[refidx][i]; - *cnt = mv_cnt[refidx]; - - mvlist[0][0] = mv_list[refidx_dav1d[0] + 1][0]; - mvlist[0][1] = mv_list[refidx_dav1d[0] + 1][1]; - if (refidx_dav1d[1] != -1) { - mvlist[1][0] = mv_list[refidx_dav1d[1] + 1][0]; - mvlist[1][1] = mv_list[refidx_dav1d[1] + 1][1]; - } - - if (ctx) { - if (refidx_dav1d[1] == -1) - *ctx = single_context[refidx_dav1d[0] + 1]; - else - *ctx = av1_mode_context_analyzer(single_context, rf); - } -} - -int dav1d_init_ref_mv_common(AV1_COMMON *cm, const int w8, const int h8, - const ptrdiff_t stride, const int allow_sb128, - MV_REF *cur, MV_REF *ref_mvs[7], - const unsigned cur_poc, - const unsigned ref_poc[7], - const unsigned ref_ref_poc[7][7], - const Dav1dWarpedMotionParams gmv[7], - const int allow_hp, const int force_int_mv, - const int allow_ref_frame_mvs, - const int order_hint); -int dav1d_init_ref_mv_common(AV1_COMMON *cm, const int w8, const int h8, - const ptrdiff_t stride, const int allow_sb128, - MV_REF *cur, MV_REF *ref_mvs[7], - const unsigned cur_poc, - const unsigned ref_poc[7], - const unsigned ref_ref_poc[7][7], - const Dav1dWarpedMotionParams gmv[7], - const int allow_hp, const int force_int_mv, - const int allow_ref_frame_mvs, - const int order_hint) -{ - if (cm->mi_cols != (w8 << 1) || cm->mi_rows != (h8 << 1)) { - const int align_h = (h8 + 15) & ~15; - if (cm->tpl_mvs) free(cm->tpl_mvs); - cm->tpl_mvs = malloc(sizeof(*cm->tpl_mvs) * (stride >> 1) * align_h); - if (!cm->tpl_mvs) { - cm->mi_cols = cm->mi_rows = 0; - return DAV1D_ERR(ENOMEM); - } - for (int i = 0; i < 7; i++) - cm->frame_refs[i].idx = i; - cm->mi_cols = w8 << 1; - cm->mi_rows = h8 << 1; - cm->mi_stride = (int) stride; - for (int i = 0; i < 7; i++) { - cm->buffer_pool.frame_bufs[i].mi_rows = cm->mi_rows; - cm->buffer_pool.frame_bufs[i].mi_cols = cm->mi_cols; - cm->buffer_pool.frame_bufs[i].mv_stride = stride; - } - cm->cur_frame.mv_stride = stride; - } - - cm->allow_high_precision_mv = allow_hp; - cm->seq_params.sb_size = allow_sb128 ? BLOCK_128X128 : BLOCK_64X64; - - cm->seq_params.enable_order_hint = !!order_hint; - cm->seq_params.order_hint_bits_minus1 = order_hint - 1; - // FIXME get these from the sequence/frame headers instead of hardcoding - cm->frame_parallel_decode = 0; - cm->cur_frame_force_integer_mv = force_int_mv; - - memcpy(&cm->global_motion[1], gmv, sizeof(*gmv) * 7); - - cm->frame_offset = cur_poc; - cm->allow_ref_frame_mvs = allow_ref_frame_mvs; - cm->cur_frame.mvs = cur; - for (int i = 0; i < 7; i++) { - cm->buffer_pool.frame_bufs[i].mvs = ref_mvs[i]; - cm->buffer_pool.frame_bufs[i].intra_only = ref_mvs[i] == NULL; - cm->buffer_pool.frame_bufs[i].cur_frame_offset = ref_poc[i]; - for (int j = 0; j < 7; j++) - cm->buffer_pool.frame_bufs[i].ref_frame_offset[j] = - ref_ref_poc[i][j]; - } - av1_setup_frame_buf_refs(cm); - for (int i = 0; i < 7; i++) { - const int ref_poc = cm->buffer_pool.frame_bufs[i].cur_frame_offset; - cm->ref_frame_sign_bias[1 + i] = get_relative_dist(cm, ref_poc, cur_poc) > 0; - } - if (allow_ref_frame_mvs) { - av1_setup_motion_field(cm); - } - - return 0; -} - -void dav1d_init_ref_mv_tile_row(AV1_COMMON *cm, - int tile_col_start4, int tile_col_end4, - int row_start4, int row_end4); -void dav1d_init_ref_mv_tile_row(AV1_COMMON *cm, - int tile_col_start4, int tile_col_end4, - int row_start4, int row_end4) -{ - RefCntBuffer *const frame_bufs = cm->buffer_pool.frame_bufs; - const int cur_order_hint = cm->cur_frame.cur_frame_offset; - int *const ref_buf_idx = cm->ref_buf_idx; - int *const ref_order_hint = cm->ref_order_hint; - - int ref_stamp = MFMV_STACK_SIZE - 1; - - if (ref_buf_idx[LAST_FRAME - LAST_FRAME] >= 0) { - const int alt_of_lst_order_hint = - frame_bufs[ref_buf_idx[LAST_FRAME - LAST_FRAME]] - .ref_frame_offset[ALTREF_FRAME - LAST_FRAME]; - - const int is_lst_overlay = - (alt_of_lst_order_hint == ref_order_hint[GOLDEN_FRAME - LAST_FRAME]); - if (!is_lst_overlay) motion_field_projection(cm, LAST_FRAME, 2, - tile_col_start4, tile_col_end4, - row_start4, row_end4); - --ref_stamp; - } - - if (get_relative_dist(cm, ref_order_hint[BWDREF_FRAME - LAST_FRAME], - cur_order_hint) > 0) { - if (motion_field_projection(cm, BWDREF_FRAME, 0, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; - } - - if (get_relative_dist(cm, ref_order_hint[ALTREF2_FRAME - LAST_FRAME], - cur_order_hint) > 0) { - if (motion_field_projection(cm, ALTREF2_FRAME, 0, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; - } - - if (get_relative_dist(cm, ref_order_hint[ALTREF_FRAME - LAST_FRAME], - cur_order_hint) > 0 && - ref_stamp >= 0) - if (motion_field_projection(cm, ALTREF_FRAME, 0, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; - - if (ref_stamp >= 0 && ref_buf_idx[LAST2_FRAME - LAST_FRAME] >= 0) - if (motion_field_projection(cm, LAST2_FRAME, 2, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; -} - -AV1_COMMON *dav1d_alloc_ref_mv_common(void); -AV1_COMMON *dav1d_alloc_ref_mv_common(void) { - AV1_COMMON *cm = malloc(sizeof(*cm)); - if (!cm) return NULL; - memset(cm, 0, sizeof(*cm)); - return cm; -} - -void dav1d_free_ref_mv_common(AV1_COMMON *cm); -void dav1d_free_ref_mv_common(AV1_COMMON *cm) { - if (cm->tpl_mvs) free(cm->tpl_mvs); - free(cm); -} diff --git a/third_party/dav1d/src/refmvs.h b/third_party/dav1d/src/refmvs.h new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/refmvs.h @@ -0,0 +1,233 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * Copyright © 2020, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_REF_MVS_H +#define DAV1D_SRC_REF_MVS_H + +#include + +#include "dav1d/headers.h" + +#include "common/intops.h" + +#include "src/intra_edge.h" +#include "src/levels.h" +#include "src/tables.h" + +#define INVALID_MV 0x80008000 + +typedef struct refmvs_temporal_block { + mv mv; + int8_t ref; +} refmvs_temporal_block; + +typedef union refmvs_refpair { + int8_t ref[2]; // [0] = 0: intra=1, [1] = -1: comp=0 + uint16_t pair; +} refmvs_refpair; + +typedef union refmvs_mvpair { + mv mv[2]; + uint64_t n; +} refmvs_mvpair; + +typedef struct refmvs_block { + refmvs_mvpair mv; + refmvs_refpair ref; + uint8_t bs, mf; // 1 = globalmv+affine, 2 = newmv +} refmvs_block; + +typedef struct refmvs_frame { + const Dav1dFrameHeader *frm_hdr; + int iw4, ih4, iw8, ih8; + int sbsz; + int use_ref_frame_mvs; + uint8_t sign_bias[7], mfmv_sign[7]; + int8_t pocdiff[7]; + uint8_t mfmv_ref[3]; + int mfmv_ref2cur[3]; + int mfmv_ref2ref[3][7]; + int n_mfmvs; + + refmvs_temporal_block *rp; + /*const*/ refmvs_temporal_block *const *rp_ref; + refmvs_temporal_block *rp_proj; + ptrdiff_t rp_stride; + + refmvs_block *r; // 35 x r_stride memory + ptrdiff_t r_stride; + int n_tile_rows, n_tile_threads; +} refmvs_frame; + +typedef struct refmvs_tile { + const refmvs_frame *rf; + refmvs_block *r[32 + 5]; + refmvs_temporal_block *rp_proj; + struct { + int start, end; + } tile_col, tile_row; +} refmvs_tile; + +typedef struct refmvs_candidate { + refmvs_mvpair mv; + int weight; +} refmvs_candidate; + +// call once per frame thread +void dav1d_refmvs_init(refmvs_frame *rf); +void dav1d_refmvs_clear(refmvs_frame *rf); + +// call once per frame +int dav1d_refmvs_init_frame(refmvs_frame *rf, + const Dav1dSequenceHeader *seq_hdr, + const Dav1dFrameHeader *frm_hdr, + const unsigned ref_poc[7], + refmvs_temporal_block *rp, + const unsigned ref_ref_poc[7][7], + /*const*/ refmvs_temporal_block *const rp_ref[7], + int n_tile_threads); + +// initialize temporal MVs; this can be done in any configuration, e.g. one +// tile/sbrow at a time, where col_{start,end}8 are the tile boundaries; or +// it can just be for the whole frame's sbrow, where col_{start,end}8 are the +// frame boundaries. row_{start,end}8 are the superblock row boundaries. +void dav1d_refmvs_load_tmvs(const refmvs_frame *rf, int tile_row_idx, + int col_start8, int col_end8, + int row_start8, int row_end8); + +// cache the current tile/sbrow (or frame/sbrow)'s projectable motion vectors +// into buffers for use in future frame's temporal MV prediction +void dav1d_refmvs_save_tmvs(const refmvs_tile *rt, + int col_start8, int col_end8, + int row_start8, int row_end8); + +// initialize tile boundaries and refmvs_block pointers for one tile/sbrow +void dav1d_refmvs_tile_sbrow_init(refmvs_tile *rt, const refmvs_frame *rf, + int tile_col_start4, int tile_col_end4, + int tile_row_start4, int tile_row_end4, + int sby, int tile_row_idx); + +// call for each block +void dav1d_refmvs_find(const refmvs_tile *rt, + refmvs_candidate mvstack[8], int *cnt, + int *ctx, const refmvs_refpair ref, enum BlockSize bs, + enum EdgeFlags edge_flags, int by4, int bx4); + +static inline void splat_oneref_mv(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs, + const enum InterPredMode mode, + const int ref, const mv mv, + const int is_interintra) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + const refmvs_block tmpl = (refmvs_block) { + .ref.ref = { ref + 1, is_interintra ? 0 : -1 }, + .mv.mv[0] = mv, + .bs = bs, + .mf = (mode == GLOBALMV && imin(bw4, bh4) >= 2) | ((mode == NEWMV) * 2), + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) + r[x] = tmpl; + } while (--bh4); +} + +static inline void splat_intrabc_mv(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs, const mv mv) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + const refmvs_block tmpl = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0] = mv, + .bs = bs, + .mf = 0, + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) { + r[x] = tmpl; + } + } while (--bh4); +} + +static inline void splat_tworef_mv(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs, + const enum CompInterPredMode mode, + const refmvs_refpair ref, + const refmvs_mvpair mv) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + assert(bw4 >= 2 && bh4 >= 2); + const refmvs_block tmpl = (refmvs_block) { + .ref.pair = ref.pair + 0x0101, + .mv = mv, + .bs = bs, + .mf = (mode == GLOBALMV_GLOBALMV) | !!((1 << mode) & (0xbc)) * 2, + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) + r[x] = tmpl; + } while (--bh4); +} + +static inline void splat_intraref(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + const refmvs_block tmpl = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0].n = INVALID_MV, + .bs = bs, + .mf = 0, + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) { + r[x] = tmpl; + } + } while (--bh4); +} + +#endif /* DAV1D_SRC_REF_MVS_H */ diff --git a/third_party/dav1d/src/refmvs.c b/third_party/dav1d/src/refmvs.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/refmvs.c @@ -0,0 +1,906 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * Copyright © 2020, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "dav1d/common.h" + +#include "common/intops.h" + +#include "src/env.h" +#include "src/refmvs.h" + +static void add_spatial_candidate(refmvs_candidate *const mvstack, int *const cnt, + const int weight, const refmvs_block *const b, + const union refmvs_refpair ref, const mv gmv[2], + int *const have_newmv_match, + int *const have_refmv_match) +{ + if (b->mv.mv[0].n == INVALID_MV) return; // intra block, no intrabc + + if (ref.ref[1] == -1) { + for (int n = 0; n < 2; n++) { + if (b->ref.ref[n] == ref.ref[0]) { + const mv cand_mv = ((b->mf & 1) && gmv[0].n != INVALID_MV) ? + gmv[0] : b->mv.mv[n]; + + const int last = *cnt; + for (int m = 0; m < last; m++) + if (mvstack[m].mv.mv[0].n == cand_mv.n) { + mvstack[m].weight += weight; + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + return; + } + + if (last < 8) { + mvstack[last].mv.mv[0] = cand_mv; + mvstack[last].weight = weight; + *cnt = last + 1; + } + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + return; + } + } + } else if (b->ref.pair == ref.pair) { + const refmvs_mvpair cand_mv = { .mv = { + [0] = ((b->mf & 1) && gmv[0].n != INVALID_MV) ? gmv[0] : b->mv.mv[0], + [1] = ((b->mf & 1) && gmv[1].n != INVALID_MV) ? gmv[1] : b->mv.mv[1], + }}; + + const int last = *cnt; + for (int n = 0; n < last; n++) + if (mvstack[n].mv.n == cand_mv.n) { + mvstack[n].weight += weight; + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + return; + } + + if (last < 8) { + mvstack[last].mv = cand_mv; + mvstack[last].weight = weight; + *cnt = last + 1; + } + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + } +} + +static int scan_row(refmvs_candidate *const mvstack, int *const cnt, + const union refmvs_refpair ref, const mv gmv[2], + const refmvs_block *b, const int bw4, const int w4, + const int max_rows, const int step, + int *const have_newmv_match, int *const have_refmv_match) +{ + const refmvs_block *cand_b = b; + const enum BlockSize first_cand_bs = cand_b->bs; + const uint8_t *const first_cand_b_dim = dav1d_block_dimensions[first_cand_bs]; + int cand_bw4 = first_cand_b_dim[0]; + int len = imax(step, imin(bw4, cand_bw4)); + + if (bw4 <= cand_bw4) { + // FIXME weight can be higher for odd blocks (bx4 & 1), but then the + // position of the first block has to be odd already, i.e. not just + // for row_offset=-3/-5 + // FIXME why can this not be cand_bw4? + const int weight = bw4 == 1 ? 2 : + imax(2, imin(2 * max_rows, first_cand_b_dim[1])); + add_spatial_candidate(mvstack, cnt, len * weight, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + return weight >> 1; + } + + for (int x = 0;;) { + // FIXME if we overhang above, we could fill a bitmask so we don't have + // to repeat the add_spatial_candidate() for the next row, but just increase + // the weight here + add_spatial_candidate(mvstack, cnt, len * 2, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + x += len; + if (x >= w4) return 1; + cand_b = &b[x]; + cand_bw4 = dav1d_block_dimensions[cand_b->bs][0]; + assert(cand_bw4 < bw4); + len = imax(step, cand_bw4); + } +} + +static int scan_col(refmvs_candidate *const mvstack, int *const cnt, + const union refmvs_refpair ref, const mv gmv[2], + /*const*/ refmvs_block *const *b, const int bh4, const int h4, + const int bx4, const int max_cols, const int step, + int *const have_newmv_match, int *const have_refmv_match) +{ + const refmvs_block *cand_b = &b[0][bx4]; + const enum BlockSize first_cand_bs = cand_b->bs; + const uint8_t *const first_cand_b_dim = dav1d_block_dimensions[first_cand_bs]; + int cand_bh4 = first_cand_b_dim[1]; + int len = imax(step, imin(bh4, cand_bh4)); + + if (bh4 <= cand_bh4) { + // FIXME weight can be higher for odd blocks (by4 & 1), but then the + // position of the first block has to be odd already, i.e. not just + // for col_offset=-3/-5 + // FIXME why can this not be cand_bh4? + const int weight = bh4 == 1 ? 2 : + imax(2, imin(2 * max_cols, first_cand_b_dim[0])); + add_spatial_candidate(mvstack, cnt, len * weight, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + return weight >> 1; + } + + for (int y = 0;;) { + // FIXME if we overhang above, we could fill a bitmask so we don't have + // to repeat the add_spatial_candidate() for the next row, but just increase + // the weight here + add_spatial_candidate(mvstack, cnt, len * 2, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + y += len; + if (y >= h4) return 1; + cand_b = &b[y][bx4]; + cand_bh4 = dav1d_block_dimensions[cand_b->bs][1]; + assert(cand_bh4 < bh4); + len = imax(step, cand_bh4); + } +} + +static inline union mv mv_projection(const union mv mv, const int num, const int den) { + static const uint16_t div_mult[32] = { + 0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, + 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, + 1024, 963, 910, 862, 819, 780, 744, 712, + 682, 655, 630, 606, 585, 564, 546, 528 + }; + assert(den > 0 && den < 32); + assert(num > -32 && num < 32); + const int dm = div_mult[den]; + const int y = mv.y * num * dm, x = mv.x * num * dm; + return (union mv) { .y = (y + 8192 + (y >> 31)) >> 14, + .x = (x + 8192 + (x >> 31)) >> 14 }; +} + +static void add_temporal_candidate(const refmvs_frame *const rf, + refmvs_candidate *const mvstack, int *const cnt, + const refmvs_temporal_block *const rb, + const union refmvs_refpair ref, int *const globalmv_ctx, + const union mv gmv[]) +{ + if (rb->mv.n == INVALID_MV) return; + + union mv mv = mv_projection(rb->mv, rf->pocdiff[ref.ref[0] - 1], rb->ref); + fix_mv_precision(rf->frm_hdr, &mv); + + const int last = *cnt; + if (ref.ref[1] == -1) { + if (globalmv_ctx) + *globalmv_ctx = (abs(mv.x - gmv[0].x) | abs(mv.y - gmv[0].y)) >= 16; + + for (int n = 0; n < last; n++) + if (mvstack[n].mv.mv[0].n == mv.n) { + mvstack[n].weight += 2; + return; + } + if (last < 8) { + mvstack[last].mv.mv[0] = mv; + mvstack[last].weight = 2; + *cnt = last + 1; + } + } else { + refmvs_mvpair mvp = { .mv = { + [0] = mv, + [1] = mv_projection(rb->mv, rf->pocdiff[ref.ref[1] - 1], rb->ref), + }}; + fix_mv_precision(rf->frm_hdr, &mvp.mv[1]); + + for (int n = 0; n < last; n++) + if (mvstack[n].mv.n == mvp.n) { + mvstack[n].weight += 2; + return; + } + if (last < 8) { + mvstack[last].mv = mvp; + mvstack[last].weight = 2; + *cnt = last + 1; + } + } +} + +static void add_compound_extended_candidate(refmvs_candidate *const same, + int *const same_count, + const refmvs_block *const cand_b, + const int sign0, const int sign1, + const union refmvs_refpair ref, + const uint8_t *const sign_bias) +{ + refmvs_candidate *const diff = &same[2]; + int *const diff_count = &same_count[2]; + + for (int n = 0; n < 2; n++) { + const int cand_ref = cand_b->ref.ref[n]; + + if (cand_ref <= 0) break; + + mv cand_mv = cand_b->mv.mv[n]; + if (cand_ref == ref.ref[0]) { + if (same_count[0] < 2) + same[same_count[0]++].mv.mv[0] = cand_mv; + if (diff_count[1] < 2) { + if (sign1 ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + diff[diff_count[1]++].mv.mv[1] = cand_mv; + } + } else if (cand_ref == ref.ref[1]) { + if (same_count[1] < 2) + same[same_count[1]++].mv.mv[1] = cand_mv; + if (diff_count[0] < 2) { + if (sign0 ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + diff[diff_count[0]++].mv.mv[0] = cand_mv; + } + } else { + mv i_cand_mv = (union mv) { + .x = -cand_mv.x, + .y = -cand_mv.y + }; + + if (diff_count[0] < 2) { + diff[diff_count[0]++].mv.mv[0] = + sign0 ^ sign_bias[cand_ref - 1] ? + i_cand_mv : cand_mv; + } + + if (diff_count[1] < 2) { + diff[diff_count[1]++].mv.mv[1] = + sign1 ^ sign_bias[cand_ref - 1] ? + i_cand_mv : cand_mv; + } + } + } +} + +static void add_single_extended_candidate(refmvs_candidate mvstack[8], int *const cnt, + const refmvs_block *const cand_b, + const int sign, const uint8_t *const sign_bias) +{ + for (int n = 0; n < 2; n++) { + const int cand_ref = cand_b->ref.ref[n]; + + if (cand_ref <= 0) break; + // we need to continue even if cand_ref == ref.ref[0], since + // the candidate could have been added as a globalmv variant, + // which changes the value + // FIXME if scan_{row,col}() returned a mask for the nearest + // edge, we could skip the appropriate ones here + + mv cand_mv = cand_b->mv.mv[n]; + if (sign ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + + int m; + const int last = *cnt; + for (m = 0; m < last; m++) + if (cand_mv.n == mvstack[m].mv.mv[0].n) + break; + if (m == last) { + mvstack[m].mv.mv[0] = cand_mv; + mvstack[m].weight = 2; // "minimal" + *cnt = last + 1; + } + } +} + +/* + * refmvs_frame allocates memory for one sbrow (32 blocks high, whole frame + * wide) of 4x4-resolution refmvs_block entries for spatial MV referencing. + * mvrefs_tile[] keeps a list of 35 (32 + 3 above) pointers into this memory, + * and each sbrow, the bottom entries (y=27/29/31) are exchanged with the top + * (-5/-3/-1) pointers by calling dav1d_refmvs_tile_sbrow_init() at the start + * of each tile/sbrow. + * + * For temporal MV referencing, we call dav1d_refmvs_save_tmvs() at the end of + * each tile/sbrow (when tile column threading is enabled), or at the start of + * each interleaved sbrow (i.e. once for all tile columns together, when tile + * column threading is disabled). This will copy the 4x4-resolution spatial MVs + * into 8x8-resolution refmvs_temporal_block structures. Then, for subsequent + * frames, at the start of each tile/sbrow (when tile column threading is + * enabled) or at the start of each interleaved sbrow (when tile column + * threading is disabled), we call load_tmvs(), which will project the MVs to + * their respective position in the current frame. + */ + +void dav1d_refmvs_find(const refmvs_tile *const rt, + refmvs_candidate mvstack[8], int *const cnt, + int *const ctx, + const union refmvs_refpair ref, const enum BlockSize bs, + const enum EdgeFlags edge_flags, + const int by4, const int bx4) +{ + const refmvs_frame *const rf = rt->rf; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], w4 = imin(imin(bw4, 16), rt->tile_col.end - bx4); + const int bh4 = b_dim[1], h4 = imin(imin(bh4, 16), rt->tile_row.end - by4); + mv gmv[2], tgmv[2]; + + *cnt = 0; + assert(ref.ref[0] >= 0 && ref.ref[0] <= 8 && + ref.ref[1] >= -1 && ref.ref[1] <= 8); + if (ref.ref[0] > 0) { + tgmv[0] = get_gmv_2d(&rf->frm_hdr->gmv[ref.ref[0] - 1], + bx4, by4, bw4, bh4, rf->frm_hdr); + gmv[0] = rf->frm_hdr->gmv[ref.ref[0] - 1].type > DAV1D_WM_TYPE_TRANSLATION ? + tgmv[0] : (mv) { .n = INVALID_MV }; + } else { + tgmv[0] = (mv) { .n = 0 }; + gmv[0] = (mv) { .n = INVALID_MV }; + } + if (ref.ref[1] > 0) { + tgmv[1] = get_gmv_2d(&rf->frm_hdr->gmv[ref.ref[1] - 1], + bx4, by4, bw4, bh4, rf->frm_hdr); + gmv[1] = rf->frm_hdr->gmv[ref.ref[1] - 1].type > DAV1D_WM_TYPE_TRANSLATION ? + tgmv[1] : (mv) { .n = INVALID_MV }; + } + + // top + int have_newmv = 0, have_col_mvs = 0, have_row_mvs = 0; + unsigned max_rows = 0, n_rows = ~0; + const refmvs_block *b_top; + if (by4 > rt->tile_row.start) { + max_rows = imin((by4 - rt->tile_row.start + 1) >> 1, 2 + (bh4 > 1)); + b_top = &rt->r[(by4 & 31) - 1 + 5][bx4]; + n_rows = scan_row(mvstack, cnt, ref, gmv, b_top, + bw4, w4, max_rows, bw4 >= 16 ? 4 : 1, + &have_newmv, &have_row_mvs); + } + + // left + unsigned max_cols = 0, n_cols = ~0U; + refmvs_block *const *b_left; + if (bx4 > rt->tile_col.start) { + max_cols = imin((bx4 - rt->tile_col.start + 1) >> 1, 2 + (bw4 > 1)); + b_left = &rt->r[(by4 & 31) + 5]; + n_cols = scan_col(mvstack, cnt, ref, gmv, b_left, + bh4, h4, bx4 - 1, max_cols, bh4 >= 16 ? 4 : 1, + &have_newmv, &have_col_mvs); + } + + // top/right + if (n_rows != ~0U && edge_flags & EDGE_I444_TOP_HAS_RIGHT && + imax(bw4, bh4) <= 16 && bw4 + bx4 < rt->tile_col.end) + { + add_spatial_candidate(mvstack, cnt, 4, &b_top[bw4], ref, gmv, + &have_newmv, &have_row_mvs); + } + + const int nearest_match = have_col_mvs + have_row_mvs; + const int nearest_cnt = *cnt; + for (int n = 0; n < nearest_cnt; n++) + mvstack[n].weight += 640; + + // temporal + int globalmv_ctx = rf->frm_hdr->use_ref_frame_mvs; + if (rf->use_ref_frame_mvs) { + const ptrdiff_t stride = rf->rp_stride; + const int by8 = by4 >> 1, bx8 = bx4 >> 1; + const refmvs_temporal_block *const rbi = &rt->rp_proj[(by8 & 15) * stride + bx8]; + const refmvs_temporal_block *rb = rbi; + const int step_h = bw4 >= 16 ? 2 : 1, step_v = bh4 >= 16 ? 2 : 1; + const int w8 = imin((w4 + 1) >> 1, 8), h8 = imin((h4 + 1) >> 1, 8); + for (int y = 0; y < h8; y += step_v) { + for (int x = 0; x < w8; x+= step_h) { + add_temporal_candidate(rf, mvstack, cnt, &rb[x], ref, + !(x | y) ? &globalmv_ctx : NULL, tgmv); + } + rb += stride * step_v; + } + if (imin(bw4, bh4) >= 2 && imax(bw4, bh4) < 16) { + const int bh8 = bh4 >> 1, bw8 = bw4 >> 1; + rb = &rbi[bh8 * stride]; + const int has_bottom = by8 + bh8 < imin(rt->tile_row.end >> 1, + (by8 & ~7) + 8); + if (has_bottom && bx8 - 1 >= imax(rt->tile_col.start >> 1, bx8 & ~7)) { + add_temporal_candidate(rf, mvstack, cnt, &rb[-1], ref, + NULL, NULL); + } + if (bx8 + bw8 < imin(rt->tile_col.end >> 1, (bx8 & ~7) + 8)) { + if (has_bottom) { + add_temporal_candidate(rf, mvstack, cnt, &rb[bw8], ref, + NULL, NULL); + } + if (by8 + bh8 - 1 < imin(rt->tile_row.end >> 1, (by8 & ~7) + 8)) { + add_temporal_candidate(rf, mvstack, cnt, &rb[bw8 - stride], + ref, NULL, NULL); + } + } + } + } + assert(*cnt <= 8); + + // top/left (which, confusingly, is part of "secondary" references) + int have_dummy_newmv_match; + if ((n_rows | n_cols) != ~0U) { + add_spatial_candidate(mvstack, cnt, 4, &b_top[-1], ref, gmv, + &have_dummy_newmv_match, &have_row_mvs); + } + + // "secondary" (non-direct neighbour) top & left edges + // what is different about secondary is that everything is now in 8x8 resolution + for (int n = 2; n <= 3; n++) { + if ((unsigned) n > n_rows && (unsigned) n <= max_rows) { + n_rows += scan_row(mvstack, cnt, ref, gmv, + &rt->r[(((by4 & 31) - 2 * n + 1) | 1) + 5][bx4 | 1], + bw4, w4, 1 + max_rows - n, bw4 >= 16 ? 4 : 2, + &have_dummy_newmv_match, &have_row_mvs); + } + + if ((unsigned) n > n_cols && (unsigned) n <= max_cols) { + n_cols += scan_col(mvstack, cnt, ref, gmv, &rt->r[((by4 & 31) | 1) + 5], + bh4, h4, (bx4 - n * 2 + 1) | 1, + 1 + max_cols - n, bh4 >= 16 ? 4 : 2, + &have_dummy_newmv_match, &have_col_mvs); + } + } + assert(*cnt <= 8); + + const int ref_match_count = have_col_mvs + have_row_mvs; + + // context build-up + int refmv_ctx, newmv_ctx; + switch (nearest_match) { + case 0: + refmv_ctx = imin(2, ref_match_count); + newmv_ctx = ref_match_count > 0; + break; + case 1: + refmv_ctx = imin(ref_match_count * 3, 4); + newmv_ctx = 3 - have_newmv; + break; + case 2: + refmv_ctx = 5; + newmv_ctx = 5 - have_newmv; + break; + } + + // sorting (nearest, then "secondary") + int len = nearest_cnt; + while (len) { + int last = 0; + for (int n = 1; n < len; n++) { + if (mvstack[n - 1].weight < mvstack[n].weight) { +#define EXCHANGE(a, b) do { refmvs_candidate tmp = a; a = b; b = tmp; } while (0) + EXCHANGE(mvstack[n - 1], mvstack[n]); + last = n; + } + } + len = last; + } + len = *cnt; + while (len > nearest_cnt) { + int last = nearest_cnt; + for (int n = nearest_cnt + 1; n < len; n++) { + if (mvstack[n - 1].weight < mvstack[n].weight) { + EXCHANGE(mvstack[n - 1], mvstack[n]); +#undef EXCHANGE + last = n; + } + } + len = last; + } + + if (ref.ref[1] > 0) { + if (*cnt < 2) { + const int sign0 = rf->sign_bias[ref.ref[0] - 1]; + const int sign1 = rf->sign_bias[ref.ref[1] - 1]; + const int sz4 = imin(w4, h4); + refmvs_candidate *const same = &mvstack[*cnt]; + int same_count[4] = { 0 }; + + // non-self references in top + if (n_rows != ~0U) for (int x = 0; x < sz4;) { + const refmvs_block *const cand_b = &b_top[x]; + add_compound_extended_candidate(same, same_count, cand_b, + sign0, sign1, ref, rf->sign_bias); + x += dav1d_block_dimensions[cand_b->bs][0]; + } + + // non-self references in left + if (n_cols != ~0U) for (int y = 0; y < sz4;) { + const refmvs_block *const cand_b = &b_left[y][bx4 - 1]; + add_compound_extended_candidate(same, same_count, cand_b, + sign0, sign1, ref, rf->sign_bias); + y += dav1d_block_dimensions[cand_b->bs][1]; + } + + refmvs_candidate *const diff = &same[2]; + const int *const diff_count = &same_count[2]; + + // merge together + for (int n = 0; n < 2; n++) { + int m = same_count[n]; + + if (m >= 2) continue; + + const int l = diff_count[n]; + if (l) { + same[m].mv.mv[n] = diff[0].mv.mv[n]; + if (++m == 2) continue; + if (l == 2) { + same[1].mv.mv[n] = diff[1].mv.mv[n]; + continue; + } + } + do { + same[m].mv.mv[n] = tgmv[n]; + } while (++m < 2); + } + + // if the first extended was the same as the non-extended one, + // then replace it with the second extended one + int n = *cnt; + if (n == 1 && mvstack[0].mv.n == same[0].mv.n) + mvstack[1].mv = mvstack[2].mv; + do { + mvstack[n].weight = 2; + } while (++n < 2); + *cnt = 2; + } + + // clamping + const int left = -(bx4 + bw4 + 4) * 4 * 8; + const int right = (rf->iw4 - bx4 + 4) * 4 * 8; + const int top = -(by4 + bh4 + 4) * 4 * 8; + const int bottom = (rf->ih4 - by4 + 4) * 4 * 8; + + const int n_refmvs = *cnt; + int n = 0; + do { + mvstack[n].mv.mv[0].x = iclip(mvstack[n].mv.mv[0].x, left, right); + mvstack[n].mv.mv[0].y = iclip(mvstack[n].mv.mv[0].y, top, bottom); + mvstack[n].mv.mv[1].x = iclip(mvstack[n].mv.mv[1].x, left, right); + mvstack[n].mv.mv[1].y = iclip(mvstack[n].mv.mv[1].y, top, bottom); + } while (++n < n_refmvs); + + switch (refmv_ctx >> 1) { + case 0: + *ctx = imin(newmv_ctx, 1); + break; + case 1: + *ctx = 1 + imin(newmv_ctx, 3); + break; + case 2: + *ctx = iclip(3 + newmv_ctx, 4, 7); + break; + } + + return; + } else if (*cnt < 2 && ref.ref[0] > 0) { + const int sign = rf->sign_bias[ref.ref[0] - 1]; + const int sz4 = imin(w4, h4); + + // non-self references in top + if (n_rows != ~0U) for (int x = 0; x < sz4 && *cnt < 2;) { + const refmvs_block *const cand_b = &b_top[x]; + add_single_extended_candidate(mvstack, cnt, cand_b, sign, rf->sign_bias); + x += dav1d_block_dimensions[cand_b->bs][0]; + } + + // non-self references in left + if (n_cols != ~0U) for (int y = 0; y < sz4 && *cnt < 2;) { + const refmvs_block *const cand_b = &b_left[y][bx4 - 1]; + add_single_extended_candidate(mvstack, cnt, cand_b, sign, rf->sign_bias); + y += dav1d_block_dimensions[cand_b->bs][1]; + } + } + assert(*cnt <= 8); + + // clamping + int n_refmvs = *cnt; + if (n_refmvs) { + const int left = -(bx4 + bw4 + 4) * 4 * 8; + const int right = (rf->iw4 - bx4 + 4) * 4 * 8; + const int top = -(by4 + bh4 + 4) * 4 * 8; + const int bottom = (rf->ih4 - by4 + 4) * 4 * 8; + + int n = 0; + do { + mvstack[n].mv.mv[0].x = iclip(mvstack[n].mv.mv[0].x, left, right); + mvstack[n].mv.mv[0].y = iclip(mvstack[n].mv.mv[0].y, top, bottom); + } while (++n < n_refmvs); + } + + for (int n = *cnt; n < 2; n++) + mvstack[n].mv.mv[0] = tgmv[0]; + + *ctx = (refmv_ctx << 4) | (globalmv_ctx << 3) | newmv_ctx; +} + +void dav1d_refmvs_tile_sbrow_init(refmvs_tile *const rt, const refmvs_frame *const rf, + const int tile_col_start4, const int tile_col_end4, + const int tile_row_start4, const int tile_row_end4, + const int sby, int tile_row_idx) +{ + if (rf->n_tile_threads == 1) tile_row_idx = 0; + rt->rp_proj = &rf->rp_proj[16 * rf->rp_stride * tile_row_idx]; + refmvs_block *r = &rf->r[35 * rf->r_stride * tile_row_idx]; + const int sbsz = rf->sbsz; + const int off = (sbsz * sby) & 16; + for (int i = 0; i < sbsz; i++, r += rf->r_stride) + rt->r[off + 5 + i] = r; + rt->r[off + 0] = r; + r += rf->r_stride; + rt->r[off + 1] = NULL; + rt->r[off + 2] = r; + r += rf->r_stride; + rt->r[off + 3] = NULL; + rt->r[off + 4] = r; + if (sby & 1) { +#define EXCHANGE(a, b) do { void *const tmp = a; a = b; b = tmp; } while (0) + EXCHANGE(rt->r[off + 0], rt->r[off + sbsz + 0]); + EXCHANGE(rt->r[off + 2], rt->r[off + sbsz + 2]); + EXCHANGE(rt->r[off + 4], rt->r[off + sbsz + 4]); +#undef EXCHANGE + } + + rt->rf = rf; + rt->tile_row.start = tile_row_start4; + rt->tile_row.end = imin(tile_row_end4, rf->ih4); + rt->tile_col.start = tile_col_start4; + rt->tile_col.end = imin(tile_col_end4, rf->iw4); +} + +void dav1d_refmvs_load_tmvs(const refmvs_frame *const rf, int tile_row_idx, + const int col_start8, const int col_end8, + const int row_start8, int row_end8) +{ + if (rf->n_tile_threads == 1) tile_row_idx = 0; + assert(row_start8 >= 0); + assert((unsigned) (row_end8 - row_start8) <= 16U); + row_end8 = imin(row_end8, rf->ih8); + const int col_start8i = imax(col_start8 - 8, 0); + const int col_end8i = imin(col_end8 + 8, rf->iw8); + + const ptrdiff_t stride = rf->rp_stride; + refmvs_temporal_block *rp_proj = + &rf->rp_proj[16 * stride * tile_row_idx + (row_start8 & 15) * stride]; + for (int y = row_start8; y < row_end8; y++) { + for (int x = col_start8; x < col_end8; x++) + rp_proj[x].mv.n = INVALID_MV; + rp_proj += stride; + } + + rp_proj = &rf->rp_proj[16 * stride * tile_row_idx]; + for (int n = 0; n < rf->n_mfmvs; n++) { + const int ref2cur = rf->mfmv_ref2cur[n]; + if (ref2cur == INT_MIN) continue; + + const int ref = rf->mfmv_ref[n]; + const int ref_sign = ref - 4; + const refmvs_temporal_block *r = &rf->rp_ref[ref][row_start8 * stride]; + for (int y = row_start8; y < row_end8; y++) { + const int y_sb_align = y & ~7; + const int y_proj_start = imax(y_sb_align, row_start8); + const int y_proj_end = imin(y_sb_align + 8, row_end8); + for (int x = col_start8i; x < col_end8i; x++) { + const refmvs_temporal_block *rb = &r[x]; + const int b_ref = rb->ref; + if (!b_ref) continue; + const int ref2ref = rf->mfmv_ref2ref[n][b_ref - 1]; + if (!ref2ref) continue; + const mv b_mv = rb->mv; + const mv offset = mv_projection(b_mv, ref2cur, ref2ref); + int pos_x = x + apply_sign(abs(offset.x) >> 6, + offset.x ^ ref_sign); + const int pos_y = y + apply_sign(abs(offset.y) >> 6, + offset.y ^ ref_sign); + if (pos_y >= y_proj_start && pos_y < y_proj_end) { + const ptrdiff_t pos = (pos_y & 15) * stride; + for (;;) { + const int x_sb_align = x & ~7; + if (pos_x >= imax(x_sb_align - 8, col_start8) && + pos_x < imin(x_sb_align + 16, col_end8)) + { + rp_proj[pos + pos_x].mv = rb->mv; + rp_proj[pos + pos_x].ref = ref2ref; + } + if (++x >= col_end8i) break; + rb++; + if (rb->ref != b_ref || rb->mv.n != b_mv.n) break; + pos_x++; + } + } else { + for (;;) { + if (++x >= col_end8i) break; + rb++; + if (rb->ref != b_ref || rb->mv.n != b_mv.n) break; + } + } + x--; + } + r += stride; + } + } +} + +void dav1d_refmvs_save_tmvs(const refmvs_tile *const rt, + const int col_start8, int col_end8, + const int row_start8, int row_end8) +{ + const refmvs_frame *const rf = rt->rf; + + assert(row_start8 >= 0); + assert((unsigned) (row_end8 - row_start8) <= 16U); + row_end8 = imin(row_end8, rf->ih8); + col_end8 = imin(col_end8, rf->iw8); + + const ptrdiff_t stride = rf->rp_stride; + const uint8_t *const ref_sign = rf->mfmv_sign; + refmvs_temporal_block *rp = &rf->rp[row_start8 * stride]; + for (int y = row_start8; y < row_end8; y++) { + const refmvs_block *const b = rt->r[6 + (y & 15) * 2]; + + for (int x = col_start8; x < col_end8;) { + const refmvs_block *const cand_b = &b[x * 2 + 1]; + const int bw8 = (dav1d_block_dimensions[cand_b->bs][0] + 1) >> 1; + + if (cand_b->ref.ref[1] > 0 && ref_sign[cand_b->ref.ref[1] - 1] && + (abs(cand_b->mv.mv[1].y) | abs(cand_b->mv.mv[1].x)) < 4096) + { + for (int n = 0; n < bw8; n++, x++) + rp[x] = (refmvs_temporal_block) { .mv = cand_b->mv.mv[1], + .ref = cand_b->ref.ref[1] }; + } else if (cand_b->ref.ref[0] > 0 && ref_sign[cand_b->ref.ref[0] - 1] && + (abs(cand_b->mv.mv[0].y) | abs(cand_b->mv.mv[0].x)) < 4096) + { + for (int n = 0; n < bw8; n++, x++) + rp[x] = (refmvs_temporal_block) { .mv = cand_b->mv.mv[0], + .ref = cand_b->ref.ref[0] }; + } else { + for (int n = 0; n < bw8; n++, x++) + rp[x].ref = 0; // "invalid" + } + } + rp += stride; + } +} + +int dav1d_refmvs_init_frame(refmvs_frame *const rf, + const Dav1dSequenceHeader *const seq_hdr, + const Dav1dFrameHeader *const frm_hdr, + const unsigned ref_poc[7], + refmvs_temporal_block *const rp, + const unsigned ref_ref_poc[7][7], + /*const*/ refmvs_temporal_block *const rp_ref[7], + const int n_tile_threads) +{ + rf->sbsz = 16 << seq_hdr->sb128; + rf->frm_hdr = frm_hdr; + rf->iw8 = (frm_hdr->width[0] + 7) >> 3; + rf->ih8 = (frm_hdr->height + 7) >> 3; + rf->iw4 = rf->iw8 << 1; + rf->ih4 = rf->ih8 << 1; + + const ptrdiff_t r_stride = ((frm_hdr->width[0] + 127) & ~127) >> 2; + const int n_tile_rows = n_tile_threads > 1 ? frm_hdr->tiling.rows : 1; + if (r_stride != rf->r_stride || n_tile_rows != rf->n_tile_rows) { + if (rf->r) free(rf->r); + rf->r = malloc(sizeof(*rf->r) * 35 * r_stride * n_tile_rows); + if (!rf->r) return DAV1D_ERR(ENOMEM); + rf->r_stride = r_stride; + } + + const ptrdiff_t rp_stride = r_stride >> 1; + if (rp_stride != rf->rp_stride || n_tile_rows != rf->n_tile_rows) { + if (rf->rp_proj) free(rf->rp_proj); + rf->rp_proj = malloc(sizeof(*rf->rp_proj) * 16 * rp_stride * n_tile_rows); + if (!rf->rp_proj) return DAV1D_ERR(ENOMEM); + rf->rp_stride = rp_stride; + } + rf->n_tile_rows = n_tile_rows; + rf->n_tile_threads = n_tile_threads; + rf->rp = rp; + rf->rp_ref = rp_ref; + const unsigned poc = frm_hdr->frame_offset; + for (int i = 0; i < 7; i++) { + const int poc_diff = get_poc_diff(seq_hdr->order_hint_n_bits, + ref_poc[i], poc); + rf->sign_bias[i] = poc_diff > 0; + rf->mfmv_sign[i] = poc_diff < 0; + rf->pocdiff[i] = iclip(get_poc_diff(seq_hdr->order_hint_n_bits, + poc, ref_poc[i]), -31, 31); + } + + // temporal MV setup + rf->n_mfmvs = 0; + if (frm_hdr->use_ref_frame_mvs && seq_hdr->order_hint_n_bits) { + int total = 2; + if (rp_ref[0] && ref_ref_poc[0][6] != ref_poc[3] /* alt-of-last != gold */) { + rf->mfmv_ref[rf->n_mfmvs++] = 0; // last + total = 3; + } + if (rp_ref[4] && get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[4], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 4; // bwd + } + if (rp_ref[5] && get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[5], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 5; // altref2 + } + if (rf->n_mfmvs < total && rp_ref[6] && + get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[6], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 6; // altref + } + if (rf->n_mfmvs < total && rp_ref[1]) + rf->mfmv_ref[rf->n_mfmvs++] = 1; // last2 + + for (int n = 0; n < rf->n_mfmvs; n++) { + const unsigned rpoc = ref_poc[rf->mfmv_ref[n]]; + const int diff1 = get_poc_diff(seq_hdr->order_hint_n_bits, + rpoc, frm_hdr->frame_offset); + if (abs(diff1) > 31) { + rf->mfmv_ref2cur[n] = INT_MIN; + } else { + rf->mfmv_ref2cur[n] = rf->mfmv_ref[n] < 4 ? -diff1 : diff1; + for (int m = 0; m < 7; m++) { + const unsigned rrpoc = ref_ref_poc[rf->mfmv_ref[n]][m]; + const int diff2 = get_poc_diff(seq_hdr->order_hint_n_bits, + rpoc, rrpoc); + // unsigned comparison also catches the < 0 case + rf->mfmv_ref2ref[n][m] = (unsigned) diff2 > 31U ? 0 : diff2; + } + } + } + } + rf->use_ref_frame_mvs = rf->n_mfmvs > 0; + + return 0; +} + +void dav1d_refmvs_init(refmvs_frame *const rf) { + rf->r = NULL; + rf->r_stride = 0; + rf->rp_proj = NULL; + rf->rp_stride = 0; +} + +void dav1d_refmvs_clear(refmvs_frame *const rf) { + if (rf->r) free(rf->r); + if (rf->rp_proj) free(rf->rp_proj); +} diff --git a/third_party/dav1d/src/tables.h b/third_party/dav1d/src/tables.h --- a/third_party/dav1d/src/tables.h +++ b/third_party/dav1d/src/tables.h @@ -103,16 +103,18 @@ (1 << BS_8x16) | (1 << BS_8x8); extern const Dav1dWarpedMotionParams dav1d_default_wm_params; +extern const int8_t dav1d_cdef_directions[12][2]; + extern const int16_t dav1d_sgr_params[16][4]; extern const uint8_t dav1d_sgr_x_by_x[256]; extern const int8_t dav1d_mc_subpel_filters[5][15][8]; extern const int8_t dav1d_mc_warp_filter[193][8]; -extern const int16_t dav1d_resize_filter[64][8]; +extern const int8_t dav1d_resize_filter[64][8]; extern const uint8_t dav1d_sm_weights[128]; extern const uint16_t dav1d_dr_intra_derivative[44]; extern const int8_t dav1d_filter_intra_taps[5][64]; diff --git a/third_party/dav1d/src/tables.c b/third_party/dav1d/src/tables.c --- a/third_party/dav1d/src/tables.c +++ b/third_party/dav1d/src/tables.c @@ -395,10 +395,25 @@ .beta = 0, .gamma = 0, .delta = 0, }; +const int8_t dav1d_cdef_directions[2 + 8 + 2 /* dir */][2 /* pass */] = { + { 1 * 12 + 0, 2 * 12 + 0 }, // 6 + { 1 * 12 + 0, 2 * 12 - 1 }, // 7 + { -1 * 12 + 1, -2 * 12 + 2 }, // 0 + { 0 * 12 + 1, -1 * 12 + 2 }, // 1 + { 0 * 12 + 1, 0 * 12 + 2 }, // 2 + { 0 * 12 + 1, 1 * 12 + 2 }, // 3 + { 1 * 12 + 1, 2 * 12 + 2 }, // 4 + { 1 * 12 + 0, 2 * 12 + 1 }, // 5 + { 1 * 12 + 0, 2 * 12 + 0 }, // 6 + { 1 * 12 + 0, 2 * 12 - 1 }, // 7 + { -1 * 12 + 1, -2 * 12 + 2 }, // 0 + { 0 * 12 + 1, -1 * 12 + 2 }, // 1 +}; + const int16_t dav1d_sgr_params[16][4] = { // r0, r1, e0, e1 { 2, 1, 140, 3236 }, { 2, 1, 112, 2158 }, { 2, 1, 93, 1618 }, { 2, 1, 80, 1438 }, { 2, 1, 70, 1295 }, { 2, 1, 58, 1177 }, { 2, 1, 47, 1079 }, { 2, 1, 37, 996 }, { 2, 1, 30, 925 }, { 2, 1, 25, 863 }, { 0, 1, -1, 2589 }, { 0, 1, -1, 1618 }, @@ -619,43 +634,43 @@ W( 0, 0, 0, - 1, 4, 127, - 3, 1 ), W( 0, 0, 0, 0, 2, 127, - 1, 0 ), // dummy (replicate row index 191) W( 0, 0, 0, 0, 2, 127, - 1, 0 ), }; -const int16_t dav1d_resize_filter[64][8] = { - { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 0, -1, 128, 2, -1, 0, 0 }, - { 0, 1, -3, 127, 4, -2, 1, 0 }, { 0, 1, -4, 127, 6, -3, 1, 0 }, - { 0, 2, -6, 126, 8, -3, 1, 0 }, { 0, 2, -7, 125, 11, -4, 1, 0 }, - { -1, 2, -8, 125, 13, -5, 2, 0 }, { -1, 3, -9, 124, 15, -6, 2, 0 }, - { -1, 3, -10, 123, 18, -6, 2, -1 }, { -1, 3, -11, 122, 20, -7, 3, -1 }, - { -1, 4, -12, 121, 22, -8, 3, -1 }, { -1, 4, -13, 120, 25, -9, 3, -1 }, - { -1, 4, -14, 118, 28, -9, 3, -1 }, { -1, 4, -15, 117, 30, -10, 4, -1 }, - { -1, 5, -16, 116, 32, -11, 4, -1 }, { -1, 5, -16, 114, 35, -12, 4, -1 }, - { -1, 5, -17, 112, 38, -12, 4, -1 }, { -1, 5, -18, 111, 40, -13, 5, -1 }, - { -1, 5, -18, 109, 43, -14, 5, -1 }, { -1, 6, -19, 107, 45, -14, 5, -1 }, - { -1, 6, -19, 105, 48, -15, 5, -1 }, { -1, 6, -19, 103, 51, -16, 5, -1 }, - { -1, 6, -20, 101, 53, -16, 6, -1 }, { -1, 6, -20, 99, 56, -17, 6, -1 }, - { -1, 6, -20, 97, 58, -17, 6, -1 }, { -1, 6, -20, 95, 61, -18, 6, -1 }, - { -2, 7, -20, 93, 64, -18, 6, -2 }, { -2, 7, -20, 91, 66, -19, 6, -1 }, - { -2, 7, -20, 88, 69, -19, 6, -1 }, { -2, 7, -20, 86, 71, -19, 6, -1 }, - { -2, 7, -20, 84, 74, -20, 7, -2 }, { -2, 7, -20, 81, 76, -20, 7, -1 }, - { -2, 7, -20, 79, 79, -20, 7, -2 }, { -1, 7, -20, 76, 81, -20, 7, -2 }, - { -2, 7, -20, 74, 84, -20, 7, -2 }, { -1, 6, -19, 71, 86, -20, 7, -2 }, - { -1, 6, -19, 69, 88, -20, 7, -2 }, { -1, 6, -19, 66, 91, -20, 7, -2 }, - { -2, 6, -18, 64, 93, -20, 7, -2 }, { -1, 6, -18, 61, 95, -20, 6, -1 }, - { -1, 6, -17, 58, 97, -20, 6, -1 }, { -1, 6, -17, 56, 99, -20, 6, -1 }, - { -1, 6, -16, 53, 101, -20, 6, -1 }, { -1, 5, -16, 51, 103, -19, 6, -1 }, - { -1, 5, -15, 48, 105, -19, 6, -1 }, { -1, 5, -14, 45, 107, -19, 6, -1 }, - { -1, 5, -14, 43, 109, -18, 5, -1 }, { -1, 5, -13, 40, 111, -18, 5, -1 }, - { -1, 4, -12, 38, 112, -17, 5, -1 }, { -1, 4, -12, 35, 114, -16, 5, -1 }, - { -1, 4, -11, 32, 116, -16, 5, -1 }, { -1, 4, -10, 30, 117, -15, 4, -1 }, - { -1, 3, -9, 28, 118, -14, 4, -1 }, { -1, 3, -9, 25, 120, -13, 4, -1 }, - { -1, 3, -8, 22, 121, -12, 4, -1 }, { -1, 3, -7, 20, 122, -11, 3, -1 }, - { -1, 2, -6, 18, 123, -10, 3, -1 }, { 0, 2, -6, 15, 124, -9, 3, -1 }, - { 0, 2, -5, 13, 125, -8, 2, -1 }, { 0, 1, -4, 11, 125, -7, 2, 0 }, - { 0, 1, -3, 8, 126, -6, 2, 0 }, { 0, 1, -3, 6, 127, -4, 1, 0 }, - { 0, 1, -2, 4, 127, -3, 1, 0 }, { 0, 0, -1, 2, 128, -1, 0, 0 }, +const int8_t ALIGN(dav1d_resize_filter[64][8], 8) = { + { 0, 0, 0, -128, 0, 0, 0, 0 }, { 0, 0, 1, -128, -2, 1, 0, 0 }, + { 0, -1, 3, -127, -4, 2, -1, 0 }, { 0, -1, 4, -127, -6, 3, -1, 0 }, + { 0, -2, 6, -126, -8, 3, -1, 0 }, { 0, -2, 7, -125, -11, 4, -1, 0 }, + { 1, -2, 8, -125, -13, 5, -2, 0 }, { 1, -3, 9, -124, -15, 6, -2, 0 }, + { 1, -3, 10, -123, -18, 6, -2, 1 }, { 1, -3, 11, -122, -20, 7, -3, 1 }, + { 1, -4, 12, -121, -22, 8, -3, 1 }, { 1, -4, 13, -120, -25, 9, -3, 1 }, + { 1, -4, 14, -118, -28, 9, -3, 1 }, { 1, -4, 15, -117, -30, 10, -4, 1 }, + { 1, -5, 16, -116, -32, 11, -4, 1 }, { 1, -5, 16, -114, -35, 12, -4, 1 }, + { 1, -5, 17, -112, -38, 12, -4, 1 }, { 1, -5, 18, -111, -40, 13, -5, 1 }, + { 1, -5, 18, -109, -43, 14, -5, 1 }, { 1, -6, 19, -107, -45, 14, -5, 1 }, + { 1, -6, 19, -105, -48, 15, -5, 1 }, { 1, -6, 19, -103, -51, 16, -5, 1 }, + { 1, -6, 20, -101, -53, 16, -6, 1 }, { 1, -6, 20, -99, -56, 17, -6, 1 }, + { 1, -6, 20, -97, -58, 17, -6, 1 }, { 1, -6, 20, -95, -61, 18, -6, 1 }, + { 2, -7, 20, -93, -64, 18, -6, 2 }, { 2, -7, 20, -91, -66, 19, -6, 1 }, + { 2, -7, 20, -88, -69, 19, -6, 1 }, { 2, -7, 20, -86, -71, 19, -6, 1 }, + { 2, -7, 20, -84, -74, 20, -7, 2 }, { 2, -7, 20, -81, -76, 20, -7, 1 }, + { 2, -7, 20, -79, -79, 20, -7, 2 }, { 1, -7, 20, -76, -81, 20, -7, 2 }, + { 2, -7, 20, -74, -84, 20, -7, 2 }, { 1, -6, 19, -71, -86, 20, -7, 2 }, + { 1, -6, 19, -69, -88, 20, -7, 2 }, { 1, -6, 19, -66, -91, 20, -7, 2 }, + { 2, -6, 18, -64, -93, 20, -7, 2 }, { 1, -6, 18, -61, -95, 20, -6, 1 }, + { 1, -6, 17, -58, -97, 20, -6, 1 }, { 1, -6, 17, -56, -99, 20, -6, 1 }, + { 1, -6, 16, -53, -101, 20, -6, 1 }, { 1, -5, 16, -51, -103, 19, -6, 1 }, + { 1, -5, 15, -48, -105, 19, -6, 1 }, { 1, -5, 14, -45, -107, 19, -6, 1 }, + { 1, -5, 14, -43, -109, 18, -5, 1 }, { 1, -5, 13, -40, -111, 18, -5, 1 }, + { 1, -4, 12, -38, -112, 17, -5, 1 }, { 1, -4, 12, -35, -114, 16, -5, 1 }, + { 1, -4, 11, -32, -116, 16, -5, 1 }, { 1, -4, 10, -30, -117, 15, -4, 1 }, + { 1, -3, 9, -28, -118, 14, -4, 1 }, { 1, -3, 9, -25, -120, 13, -4, 1 }, + { 1, -3, 8, -22, -121, 12, -4, 1 }, { 1, -3, 7, -20, -122, 11, -3, 1 }, + { 1, -2, 6, -18, -123, 10, -3, 1 }, { 0, -2, 6, -15, -124, 9, -3, 1 }, + { 0, -2, 5, -13, -125, 8, -2, 1 }, { 0, -1, 4, -11, -125, 7, -2, 0 }, + { 0, -1, 3, -8, -126, 6, -2, 0 }, { 0, -1, 3, -6, -127, 4, -1, 0 }, + { 0, -1, 2, -4, -127, 3, -1, 0 }, { 0, 0, 1, -2, -128, 1, 0, 0 }, }; const uint8_t dav1d_sm_weights[128] = { // Unused, because we always offset by bs, which is at least 2. 0, 0, diff --git a/third_party/dav1d/src/thread.h b/third_party/dav1d/src/thread.h --- a/third_party/dav1d/src/thread.h +++ b/third_party/dav1d/src/thread.h @@ -28,10 +28,11 @@ #ifndef DAV1D_SRC_THREAD_H #define DAV1D_SRC_THREAD_H #if defined(_WIN32) +#include #include #define PTHREAD_ONCE_INIT INIT_ONCE_STATIC_INIT typedef struct { @@ -70,13 +71,14 @@ static inline int pthread_attr_destroy(pthread_attr_t *const attr) { return 0; } static inline int pthread_attr_setstacksize(pthread_attr_t *const attr, - const unsigned stack_size) + const size_t stack_size) { - attr->stack_size = stack_size; + if (stack_size > UINT_MAX) return 1; + attr->stack_size = (unsigned) stack_size; return 0; } static inline int pthread_mutex_init(pthread_mutex_t *const mutex, const void *const attr) diff --git a/third_party/dav1d/src/thread_task.c b/third_party/dav1d/src/thread_task.c --- a/third_party/dav1d/src/thread_task.c +++ b/third_party/dav1d/src/thread_task.c @@ -40,12 +40,11 @@ &f->frame_thread.td.lock); } if (f->frame_thread.die) break; pthread_mutex_unlock(&f->frame_thread.td.lock); - const int res = dav1d_decode_frame(f); - if (res) + if (dav1d_decode_frame(f)) memset(f->frame_thread.cf, 0, (size_t)f->frame_thread.cf_sz * 128 * 128 / 2); pthread_mutex_lock(&f->frame_thread.td.lock); f->n_tile_data = 0; @@ -90,12 +89,12 @@ // loopfilter happens in the main thread Dav1dTileState *const ts = t->ts = &f->ts[task_idx]; for (t->by = ts->tiling.row_start; t->by < ts->tiling.row_end; t->by += f->sb_step) { - int error = dav1d_decode_tile_sbrow(t); - int progress = error ? TILE_ERROR : 1 + (t->by >> f->sb_shift); + const int error = dav1d_decode_tile_sbrow(t); + const int progress = error ? TILE_ERROR : 1 + (t->by >> f->sb_shift); // signal progress pthread_mutex_lock(&ts->tile_thread.lock); atomic_store(&ts->progress, progress); pthread_cond_signal(&ts->tile_thread.cond); @@ -126,11 +125,11 @@ // we need to interleave sbrow decoding for all tile cols in a // tile row, since otherwise subsequent threads will be blocked // waiting for the post-filter to complete t->ts = ts; t->by = sby << f->sb_shift; - int error = dav1d_decode_tile_sbrow(t); + const int error = dav1d_decode_tile_sbrow(t); progress = error ? TILE_ERROR : 1 + sby; // signal progress pthread_mutex_lock(&ts->tile_thread.lock); atomic_store(&ts->progress, progress); diff --git a/third_party/dav1d/src/warpmv.h b/third_party/dav1d/src/warpmv.h --- a/third_party/dav1d/src/warpmv.h +++ b/third_party/dav1d/src/warpmv.h @@ -30,8 +30,10 @@ #include "src/levels.h" int dav1d_get_shear_params(Dav1dWarpedMotionParams *wm); int dav1d_find_affine_int(const int (*pts)[2][2], int np, int bw4, int bh4, - mv mv, Dav1dWarpedMotionParams *wm, int by, int bx); + mv mv, Dav1dWarpedMotionParams *wm, int bx, int by); +void dav1d_set_affine_mv2d(int bw4, int bh4, + mv mv, Dav1dWarpedMotionParams *wm, int bx, int by); #endif /* DAV1D_SRC_WARPMV_H */ diff --git a/third_party/dav1d/src/warpmv.c b/third_party/dav1d/src/warpmv.c --- a/third_party/dav1d/src/warpmv.c +++ b/third_party/dav1d/src/warpmv.c @@ -128,10 +128,26 @@ ((1LL << shift) >> 1)) >> shift), v1); return iclip(v2, 0xe001, 0x11fff); } +void dav1d_set_affine_mv2d(const int bw4, const int bh4, + const mv mv, Dav1dWarpedMotionParams *const wm, + const int bx4, const int by4) +{ + int32_t *const mat = wm->matrix; + const int rsuy = 2 * bh4 - 1; + const int rsux = 2 * bw4 - 1; + const int isuy = by4 * 4 + rsuy; + const int isux = bx4 * 4 + rsux; + + mat[0] = iclip(mv.x * 0x2000 - (isux * (mat[2] - 0x10000) + isuy * mat[3]), + -0x800000, 0x7fffff); + mat[1] = iclip(mv.y * 0x2000 - (isux * mat[4] + isuy * (mat[5] - 0x10000)), + -0x800000, 0x7fffff); +} + int dav1d_find_affine_int(const int (*pts)[2][2], const int np, const int bw4, const int bh4, const mv mv, Dav1dWarpedMotionParams *const wm, const int bx4, const int by4) { diff --git a/third_party/dav1d/src/wedge.c b/third_party/dav1d/src/wedge.c --- a/third_party/dav1d/src/wedge.c +++ b/third_party/dav1d/src/wedge.c @@ -113,11 +113,11 @@ static uint8_t ALIGN(wedge_masks_420_4x8 [2 * 16 * 4 * 8], 32); static uint8_t ALIGN(wedge_masks_420_4x4 [2 * 16 * 4 * 4], 32); const uint8_t *dav1d_wedge_masks[N_BS_SIZES][3][2][16]; -static void insert_border(uint8_t *const dst, const uint8_t *src, +static void insert_border(uint8_t *const dst, const uint8_t *const src, const int ctr) { if (ctr > 4) memset(dst, 0, ctr - 4); memcpy(dst + imax(ctr, 4) - 4, src + imax(4 - ctr, 0), imin(64 - ctr, 8)); if (ctr < 64 - 4) @@ -154,11 +154,12 @@ dst += w; } } static COLD void init_chroma(uint8_t *chroma, const uint8_t *luma, - const int sign, const int w, const int h, const int ss_ver) + const int sign, const int w, const int h, + const int ss_ver) { for (int y = 0; y < h; y += 1 + ss_ver) { for (int x = 0; x < w; x += 2) { int sum = luma[x] + luma[x + 1] + 1; if (ss_ver) sum += luma[w + x] + luma[w + x + 1] + 1; diff --git a/third_party/dav1d/src/x86/cdef.asm b/third_party/dav1d/src/x86/cdef.asm deleted file mode 100644 --- a/third_party/dav1d/src/x86/cdef.asm +++ /dev/null @@ -1,685 +0,0 @@ -; Copyright © 2018, VideoLAN and dav1d authors -; Copyright © 2018, Two Orioles, LLC -; All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions are met: -; -; 1. Redistributions of source code must retain the above copyright notice, this -; list of conditions and the following disclaimer. -; -; 2. Redistributions in binary form must reproduce the above copyright notice, -; this list of conditions and the following disclaimer in the documentation -; and/or other materials provided with the distribution. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -%include "ext/x86/x86inc.asm" - -%if ARCH_X86_64 - -SECTION_RODATA 32 -pd_47130256: dd 4, 7, 1, 3, 0, 2, 5, 6 -div_table: dd 840, 420, 280, 210, 168, 140, 120, 105 - dd 420, 210, 140, 105 -shufw_6543210x: db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 -shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 -pw_128: times 2 dw 128 -pw_2048: times 2 dw 2048 -tap_table: ; masks for 8 bit shifts - db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 - ; weights - db 4, 2, 3, 3, 2, 1 - db -1 * 16 + 1, -2 * 16 + 2 - db 0 * 16 + 1, -1 * 16 + 2 - db 0 * 16 + 1, 0 * 16 + 2 - db 0 * 16 + 1, 1 * 16 + 2 - db 1 * 16 + 1, 2 * 16 + 2 - db 1 * 16 + 0, 2 * 16 + 1 - db 1 * 16 + 0, 2 * 16 + 0 - db 1 * 16 + 0, 2 * 16 - 1 - ; the last 6 are repeats of the first 6 so we don't need to & 7 - db -1 * 16 + 1, -2 * 16 + 2 - db 0 * 16 + 1, -1 * 16 + 2 - db 0 * 16 + 1, 0 * 16 + 2 - db 0 * 16 + 1, 1 * 16 + 2 - db 1 * 16 + 1, 2 * 16 + 2 - db 1 * 16 + 0, 2 * 16 + 1 - -SECTION .text - -%macro ACCUMULATE_TAP 7 ; tap_offset, shift, mask, strength, mul_tap, w, stride - ; load p0/p1 - movsx offq, byte [dirq+kq+%1] ; off1 -%if %6 == 4 - movq xm5, [stkq+offq*2+%7*0] ; p0 - movq xm6, [stkq+offq*2+%7*2] - movhps xm5, [stkq+offq*2+%7*1] - movhps xm6, [stkq+offq*2+%7*3] - vinserti128 m5, xm6, 1 -%else - movu xm5, [stkq+offq*2+%7*0] ; p0 - vinserti128 m5, [stkq+offq*2+%7*1], 1 -%endif - neg offq ; -off1 -%if %6 == 4 - movq xm6, [stkq+offq*2+%7*0] ; p1 - movq xm9, [stkq+offq*2+%7*2] - movhps xm6, [stkq+offq*2+%7*1] - movhps xm9, [stkq+offq*2+%7*3] - vinserti128 m6, xm9, 1 -%else - movu xm6, [stkq+offq*2+%7*0] ; p1 - vinserti128 m6, [stkq+offq*2+%7*1], 1 -%endif - ; out of bounds values are set to a value that is a both a large unsigned - ; value and a negative signed value. - ; use signed max and unsigned min to remove them - pmaxsw m7, m5 ; max after p0 - pminuw m8, m5 ; min after p0 - pmaxsw m7, m6 ; max after p1 - pminuw m8, m6 ; min after p1 - - ; accumulate sum[m15] over p0/p1 - ; calculate difference before converting - psubw m5, m4 ; diff_p0(p0 - px) - psubw m6, m4 ; diff_p1(p1 - px) - - ; convert to 8-bits with signed saturation - ; saturating to large diffs has no impact on the results - packsswb m5, m6 - - ; group into pairs so we can accumulate using maddubsw - pshufb m5, m12 - pabsb m9, m5 - psignb m10, %5, m5 - psrlw m5, m9, %2 ; emulate 8-bit shift - pand m5, %3 - psubusb m5, %4, m5 - - ; use unsigned min since abs diff can equal 0x80 - pminub m5, m9 - pmaddubsw m5, m10 - paddw m15, m5 -%endmacro - -%macro CDEF_FILTER 3 ; w, h, stride -INIT_YMM avx2 -%if %1 != 4 || %2 != 8 -cglobal cdef_filter_%1x%2, 4, 9, 16, 2 * 16 + (%2+4)*%3, \ - dst, stride, left, top, pri, sec, stride3, dst4, edge -%else -cglobal cdef_filter_%1x%2, 4, 10, 16, 2 * 16 + (%2+4)*%3, \ - dst, stride, left, top, pri, sec, stride3, dst4, edge -%endif -%define px rsp+2*16+2*%3 - pcmpeqw m14, m14 - psllw m14, 15 ; 0x8000 - mov edged, r8m - - ; prepare pixel buffers - body/right -%if %1 == 4 - INIT_XMM avx2 -%endif -%if %2 == 8 - lea dst4q, [dstq+strideq*4] -%endif - lea stride3q, [strideq*3] - test edgeb, 2 ; have_right - jz .no_right - pmovzxbw m1, [dstq+strideq*0] - pmovzxbw m2, [dstq+strideq*1] - pmovzxbw m3, [dstq+strideq*2] - pmovzxbw m4, [dstq+stride3q] - mova [px+0*%3], m1 - mova [px+1*%3], m2 - mova [px+2*%3], m3 - mova [px+3*%3], m4 -%if %2 == 8 - pmovzxbw m1, [dst4q+strideq*0] - pmovzxbw m2, [dst4q+strideq*1] - pmovzxbw m3, [dst4q+strideq*2] - pmovzxbw m4, [dst4q+stride3q] - mova [px+4*%3], m1 - mova [px+5*%3], m2 - mova [px+6*%3], m3 - mova [px+7*%3], m4 -%endif - jmp .body_done -.no_right: -%if %1 == 4 - movd xm1, [dstq+strideq*0] - movd xm2, [dstq+strideq*1] - movd xm3, [dstq+strideq*2] - movd xm4, [dstq+stride3q] - pmovzxbw xm1, xm1 - pmovzxbw xm2, xm2 - pmovzxbw xm3, xm3 - pmovzxbw xm4, xm4 - movq [px+0*%3], xm1 - movq [px+1*%3], xm2 - movq [px+2*%3], xm3 - movq [px+3*%3], xm4 -%else - pmovzxbw xm1, [dstq+strideq*0] - pmovzxbw xm2, [dstq+strideq*1] - pmovzxbw xm3, [dstq+strideq*2] - pmovzxbw xm4, [dstq+stride3q] - mova [px+0*%3], xm1 - mova [px+1*%3], xm2 - mova [px+2*%3], xm3 - mova [px+3*%3], xm4 -%endif - movd [px+0*%3+%1*2], xm14 - movd [px+1*%3+%1*2], xm14 - movd [px+2*%3+%1*2], xm14 - movd [px+3*%3+%1*2], xm14 -%if %2 == 8 - %if %1 == 4 - movd xm1, [dst4q+strideq*0] - movd xm2, [dst4q+strideq*1] - movd xm3, [dst4q+strideq*2] - movd xm4, [dst4q+stride3q] - pmovzxbw xm1, xm1 - pmovzxbw xm2, xm2 - pmovzxbw xm3, xm3 - pmovzxbw xm4, xm4 - movq [px+4*%3], xm1 - movq [px+5*%3], xm2 - movq [px+6*%3], xm3 - movq [px+7*%3], xm4 - %else - pmovzxbw xm1, [dst4q+strideq*0] - pmovzxbw xm2, [dst4q+strideq*1] - pmovzxbw xm3, [dst4q+strideq*2] - pmovzxbw xm4, [dst4q+stride3q] - mova [px+4*%3], xm1 - mova [px+5*%3], xm2 - mova [px+6*%3], xm3 - mova [px+7*%3], xm4 - %endif - movd [px+4*%3+%1*2], xm14 - movd [px+5*%3+%1*2], xm14 - movd [px+6*%3+%1*2], xm14 - movd [px+7*%3+%1*2], xm14 -%endif -.body_done: - - ; top - DEFINE_ARGS dst, stride, left, top2, pri, sec, stride3, top1, edge - test edgeb, 4 ; have_top - jz .no_top - mov top1q, [top2q+0*gprsize] - mov top2q, [top2q+1*gprsize] - test edgeb, 1 ; have_left - jz .top_no_left - test edgeb, 2 ; have_right - jz .top_no_right - pmovzxbw m1, [top1q-(%1/2)] - pmovzxbw m2, [top2q-(%1/2)] - movu [px-2*%3-%1], m1 - movu [px-1*%3-%1], m2 - jmp .top_done -.top_no_right: - pmovzxbw m1, [top1q-%1] - pmovzxbw m2, [top2q-%1] - movu [px-2*%3-%1*2], m1 - movu [px-1*%3-%1*2], m2 - movd [px-2*%3+%1*2], xm14 - movd [px-1*%3+%1*2], xm14 - jmp .top_done -.top_no_left: - test edgeb, 2 ; have_right - jz .top_no_left_right - pmovzxbw m1, [top1q] - pmovzxbw m2, [top2q] - mova [px-2*%3+0], m1 - mova [px-1*%3+0], m2 - movd [px-2*%3-4], xm14 - movd [px-1*%3-4], xm14 - jmp .top_done -.top_no_left_right: -%if %1 == 4 - movd xm1, [top1q] - pinsrd xm1, [top2q], 1 - pmovzxbw xm1, xm1 - movq [px-2*%3+0], xm1 - movhps [px-1*%3+0], xm1 -%else - pmovzxbw xm1, [top1q] - pmovzxbw xm2, [top2q] - mova [px-2*%3+0], xm1 - mova [px-1*%3+0], xm2 -%endif - movd [px-2*%3-4], xm14 - movd [px-1*%3-4], xm14 - movd [px-2*%3+%1*2], xm14 - movd [px-1*%3+%1*2], xm14 - jmp .top_done -.no_top: - movu [px-2*%3-%1], m14 - movu [px-1*%3-%1], m14 -.top_done: - - ; left - test edgeb, 1 ; have_left - jz .no_left - pmovzxbw xm1, [leftq+ 0] -%if %2 == 8 - pmovzxbw xm2, [leftq+ 8] -%endif - movd [px+0*%3-4], xm1 - pextrd [px+1*%3-4], xm1, 1 - pextrd [px+2*%3-4], xm1, 2 - pextrd [px+3*%3-4], xm1, 3 -%if %2 == 8 - movd [px+4*%3-4], xm2 - pextrd [px+5*%3-4], xm2, 1 - pextrd [px+6*%3-4], xm2, 2 - pextrd [px+7*%3-4], xm2, 3 -%endif - jmp .left_done -.no_left: - movd [px+0*%3-4], xm14 - movd [px+1*%3-4], xm14 - movd [px+2*%3-4], xm14 - movd [px+3*%3-4], xm14 -%if %2 == 8 - movd [px+4*%3-4], xm14 - movd [px+5*%3-4], xm14 - movd [px+6*%3-4], xm14 - movd [px+7*%3-4], xm14 -%endif -.left_done: - - ; bottom - DEFINE_ARGS dst, stride, dst8, dummy1, pri, sec, stride3, dummy3, edge - test edgeb, 8 ; have_bottom - jz .no_bottom - lea dst8q, [dstq+%2*strideq] - test edgeb, 1 ; have_left - jz .bottom_no_left - test edgeb, 2 ; have_right - jz .bottom_no_right - pmovzxbw m1, [dst8q-(%1/2)] - pmovzxbw m2, [dst8q+strideq-(%1/2)] - movu [px+(%2+0)*%3-%1], m1 - movu [px+(%2+1)*%3-%1], m2 - jmp .bottom_done -.bottom_no_right: - pmovzxbw m1, [dst8q-%1] - pmovzxbw m2, [dst8q+strideq-%1] - movu [px+(%2+0)*%3-%1*2], m1 - movu [px+(%2+1)*%3-%1*2], m2 -%if %1 == 8 - movd [px+(%2-1)*%3+%1*2], xm14 ; overwritten by previous movu -%endif - movd [px+(%2+0)*%3+%1*2], xm14 - movd [px+(%2+1)*%3+%1*2], xm14 - jmp .bottom_done -.bottom_no_left: - test edgeb, 2 ; have_right - jz .bottom_no_left_right - pmovzxbw m1, [dst8q] - pmovzxbw m2, [dst8q+strideq] - mova [px+(%2+0)*%3+0], m1 - mova [px+(%2+1)*%3+0], m2 - movd [px+(%2+0)*%3-4], xm14 - movd [px+(%2+1)*%3-4], xm14 - jmp .bottom_done -.bottom_no_left_right: -%if %1 == 4 - movd xm1, [dst8q] - pinsrd xm1, [dst8q+strideq], 1 - pmovzxbw xm1, xm1 - movq [px+(%2+0)*%3+0], xm1 - movhps [px+(%2+1)*%3+0], xm1 -%else - pmovzxbw xm1, [dst8q] - pmovzxbw xm2, [dst8q+strideq] - mova [px+(%2+0)*%3+0], xm1 - mova [px+(%2+1)*%3+0], xm2 -%endif - movd [px+(%2+0)*%3-4], xm14 - movd [px+(%2+1)*%3-4], xm14 - movd [px+(%2+0)*%3+%1*2], xm14 - movd [px+(%2+1)*%3+%1*2], xm14 - jmp .bottom_done -.no_bottom: - movu [px+(%2+0)*%3-%1], m14 - movu [px+(%2+1)*%3-%1], m14 -.bottom_done: - - ; actual filter - INIT_YMM avx2 - DEFINE_ARGS dst, stride, pridmp, damping, pri, secdmp, stride3, zero -%undef edged - ; register to shuffle values into after packing - vbroadcasti128 m12, [shufb_lohi] - - movifnidn prid, prim - mov dampingd, r7m - lzcnt pridmpd, prid -%if UNIX64 - movd xm0, prid - movd xm1, secdmpd -%endif - lzcnt secdmpd, secdmpm - sub dampingd, 31 - xor zerod, zerod - add pridmpd, dampingd - cmovl pridmpd, zerod - add secdmpd, dampingd - cmovl secdmpd, zerod - mov [rsp+0], pridmpq ; pri_shift - mov [rsp+8], secdmpq ; sec_shift - - DEFINE_ARGS dst, stride, pridmp, table, pri, secdmp, stride3 - lea tableq, [tap_table] - vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask - vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask - - ; pri/sec_taps[k] [4 total] - DEFINE_ARGS dst, stride, dir, table, pri, sec, stride3 -%if UNIX64 - vpbroadcastb m0, xm0 ; pri_strength - vpbroadcastb m1, xm1 ; sec_strength -%else - vpbroadcastb m0, prim - vpbroadcastb m1, secm -%endif - and prid, 1 - lea priq, [tableq+priq*2+8] ; pri_taps - lea secq, [tableq+12] ; sec_taps - - ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] - mov dird, r6m - lea dirq, [tableq+dirq*2+14] -%if %1*%2*2/mmsize > 1 - %if %1 == 4 - DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, h, off, k - %else - DEFINE_ARGS dst, stride, dir, stk, pri, sec, h, off, k - %endif - mov hd, %1*%2*2/mmsize -%else - DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, off, k -%endif - lea stkq, [px] - pxor m11, m11 -%if %1*%2*2/mmsize > 1 -.v_loop: -%endif - mov kd, 1 -%if %1 == 4 - movq xm4, [stkq+%3*0] - movhps xm4, [stkq+%3*1] - movq xm5, [stkq+%3*2] - movhps xm5, [stkq+%3*3] - vinserti128 m4, xm5, 1 -%else - mova xm4, [stkq+%3*0] ; px - vinserti128 m4, [stkq+%3*1], 1 -%endif - pxor m15, m15 ; sum - mova m7, m4 ; max - mova m8, m4 ; min -.k_loop: - vpbroadcastb m2, [priq+kq] ; pri_taps - vpbroadcastb m3, [secq+kq] ; sec_taps - - ACCUMULATE_TAP 0*2, [rsp+0], m13, m0, m2, %1, %3 - ACCUMULATE_TAP 2*2, [rsp+8], m14, m1, m3, %1, %3 - ACCUMULATE_TAP 6*2, [rsp+8], m14, m1, m3, %1, %3 - - dec kq - jge .k_loop - - vpbroadcastd m10, [pw_2048] - pcmpgtw m9, m11, m15 - paddw m15, m9 - pmulhrsw m15, m10 - paddw m4, m15 - pminsw m4, m7 - pmaxsw m4, m8 - packuswb m4, m4 - vextracti128 xm5, m4, 1 -%if %1 == 4 - movd [dstq+strideq*0], xm4 - pextrd [dstq+strideq*1], xm4, 1 - movd [dstq+strideq*2], xm5 - pextrd [dstq+stride3q], xm5, 1 -%else - movq [dstq+strideq*0], xm4 - movq [dstq+strideq*1], xm5 -%endif - -%if %1*%2*2/mmsize > 1 - %define vloop_lines (mmsize/(%1*2)) - lea dstq, [dstq+strideq*vloop_lines] - add stkq, %3*vloop_lines - dec hd - jg .v_loop -%endif - - RET -%endmacro - -CDEF_FILTER 8, 8, 32 -CDEF_FILTER 4, 8, 32 -CDEF_FILTER 4, 4, 32 - -INIT_YMM avx2 -cglobal cdef_dir, 3, 4, 15, src, stride, var, stride3 - lea stride3q, [strideq*3] - movq xm0, [srcq+strideq*0] - movq xm1, [srcq+strideq*1] - movq xm2, [srcq+strideq*2] - movq xm3, [srcq+stride3q] - lea srcq, [srcq+strideq*4] - vpbroadcastq m4, [srcq+strideq*0] - vpbroadcastq m5, [srcq+strideq*1] - vpbroadcastq m6, [srcq+strideq*2] - vpbroadcastq m7, [srcq+stride3q] - vpbroadcastd m8, [pw_128] - pxor m9, m9 - - vpblendd m0, m0, m7, 0xf0 - vpblendd m1, m1, m6, 0xf0 - vpblendd m2, m2, m5, 0xf0 - vpblendd m3, m3, m4, 0xf0 - - punpcklbw m0, m9 - punpcklbw m1, m9 - punpcklbw m2, m9 - punpcklbw m3, m9 - - psubw m0, m8 - psubw m1, m8 - psubw m2, m8 - psubw m3, m8 - - ; shuffle registers to generate partial_sum_diag[0-1] together - vpermq m7, m0, q1032 - vpermq m6, m1, q1032 - vpermq m5, m2, q1032 - vpermq m4, m3, q1032 - - ; start with partial_sum_hv[0-1] - paddw m8, m0, m1 - paddw m9, m2, m3 - phaddw m10, m0, m1 - phaddw m11, m2, m3 - paddw m8, m9 - phaddw m10, m11 - vextracti128 xm9, m8, 1 - vextracti128 xm11, m10, 1 - paddw xm8, xm9 ; partial_sum_hv[1] - phaddw xm10, xm11 ; partial_sum_hv[0] - vinserti128 m8, xm10, 1 - vpbroadcastd m9, [div_table+44] - pmaddwd m8, m8 - pmulld m8, m9 ; cost6[2a-d] | cost2[a-d] - - ; create aggregates [lower half]: - ; m9 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234+ - ; m4:xxxx0123+m5:xxxxx012+m6:xxxxxx01+m7:xxxxxxx0 - ; m10= m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx+ - ; m4:4567xxxx+m5:34567xxx+m6:234567xx+m7:1234567x - ; and [upper half]: - ; m9 = m0:xxxxxxx0+m1:xxxxxx01+m2:xxxxx012+m3:xxxx0123+ - ; m4:xxx01234+m5:xx012345+m6:x0123456+m7:01234567 - ; m10= m0:1234567x+m1:234567xx+m2:34567xxx+m3:4567xxxx+ - ; m4:567xxxxx+m5:67xxxxxx+m6:7xxxxxxx - ; and then shuffle m11 [shufw_6543210x], unpcklwd, pmaddwd, pmulld, paddd - - pslldq m9, m1, 2 - psrldq m10, m1, 14 - pslldq m11, m2, 4 - psrldq m12, m2, 12 - pslldq m13, m3, 6 - psrldq m14, m3, 10 - paddw m9, m11 - paddw m10, m12 - paddw m9, m13 - paddw m10, m14 - pslldq m11, m4, 8 - psrldq m12, m4, 8 - pslldq m13, m5, 10 - psrldq m14, m5, 6 - paddw m9, m11 - paddw m10, m12 - paddw m9, m13 - paddw m10, m14 - pslldq m11, m6, 12 - psrldq m12, m6, 4 - pslldq m13, m7, 14 - psrldq m14, m7, 2 - paddw m9, m11 - paddw m10, m12 - paddw m9, m13 - paddw m10, m14 ; partial_sum_diag[0/1][8-14,zero] - vbroadcasti128 m14, [shufw_6543210x] - vbroadcasti128 m13, [div_table+16] - vbroadcasti128 m12, [div_table+0] - paddw m9, m0 ; partial_sum_diag[0/1][0-7] - pshufb m10, m14 - punpckhwd m11, m9, m10 - punpcklwd m9, m10 - pmaddwd m11, m11 - pmaddwd m9, m9 - pmulld m11, m13 - pmulld m9, m12 - paddd m9, m11 ; cost0[a-d] | cost4[a-d] - - ; merge horizontally and vertically for partial_sum_alt[0-3] - paddw m10, m0, m1 - paddw m11, m2, m3 - paddw m12, m4, m5 - paddw m13, m6, m7 - phaddw m0, m4 - phaddw m1, m5 - phaddw m2, m6 - phaddw m3, m7 - - ; create aggregates [lower half]: - ; m4 = m10:01234567+m11:x0123456+m12:xx012345+m13:xxx01234 - ; m11= m11:7xxxxxxx+m12:67xxxxxx+m13:567xxxxx - ; and [upper half]: - ; m4 = m10:xxx01234+m11:xx012345+m12:x0123456+m13:01234567 - ; m11= m10:567xxxxx+m11:67xxxxxx+m12:7xxxxxxx - ; and then pshuflw m11 3012, unpcklwd, pmaddwd, pmulld, paddd - - pslldq m4, m11, 2 - psrldq m11, 14 - pslldq m5, m12, 4 - psrldq m12, 12 - pslldq m6, m13, 6 - psrldq m13, 10 - paddw m4, m10 - paddw m11, m12 - vpbroadcastd m12, [div_table+44] - paddw m5, m6 - paddw m11, m13 ; partial_sum_alt[3/2] right - vbroadcasti128 m13, [div_table+32] - paddw m4, m5 ; partial_sum_alt[3/2] left - pshuflw m5, m11, q3012 - punpckhwd m6, m11, m4 - punpcklwd m4, m5 - pmaddwd m6, m6 - pmaddwd m4, m4 - pmulld m6, m12 - pmulld m4, m13 - paddd m4, m6 ; cost7[a-d] | cost5[a-d] - - ; create aggregates [lower half]: - ; m5 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234 - ; m1 = m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx - ; and [upper half]: - ; m5 = m0:xxx01234+m1:xx012345+m2:x0123456+m3:01234567 - ; m1 = m0:567xxxxx+m1:67xxxxxx+m2:7xxxxxxx - ; and then pshuflw m1 3012, unpcklwd, pmaddwd, pmulld, paddd - - pslldq m5, m1, 2 - psrldq m1, 14 - pslldq m6, m2, 4 - psrldq m2, 12 - pslldq m7, m3, 6 - psrldq m3, 10 - paddw m5, m0 - paddw m1, m2 - paddw m6, m7 - paddw m1, m3 ; partial_sum_alt[0/1] right - paddw m5, m6 ; partial_sum_alt[0/1] left - pshuflw m0, m1, q3012 - punpckhwd m1, m5 - punpcklwd m5, m0 - pmaddwd m1, m1 - pmaddwd m5, m5 - pmulld m1, m12 - pmulld m5, m13 - paddd m5, m1 ; cost1[a-d] | cost3[a-d] - - mova xm0, [pd_47130256+ 16] - mova m1, [pd_47130256] - phaddd m9, m8 - phaddd m5, m4 - phaddd m9, m5 - vpermd m0, m9 ; cost[0-3] - vpermd m1, m9 ; cost[4-7] | cost[0-3] - - ; now find the best cost - pmaxsd xm2, xm0, xm1 - pshufd xm3, xm2, q1032 - pmaxsd xm2, xm3 - pshufd xm3, xm2, q2301 - pmaxsd xm2, xm3 ; best cost - - ; find the idx using minpos - ; make everything other than the best cost negative via subtraction - ; find the min of unsigned 16-bit ints to sort out the negative values - psubd xm4, xm1, xm2 - psubd xm3, xm0, xm2 - packssdw xm3, xm4 - phminposuw xm3, xm3 - - ; convert idx to 32-bits - psrld xm3, 16 - movd eax, xm3 - - ; get idx^4 complement - vpermd m3, m1 - psubd xm2, xm3 - psrld xm2, 10 - movd [varq], xm2 - RET -%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_avx2.asm b/third_party/dav1d/src/x86/cdef_avx2.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef_avx2.asm @@ -0,0 +1,1798 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +%macro JMP_TABLE 2-* + %xdefine %1_jmptable %%table + %xdefine %%base mangle(private_prefix %+ _%1_avx2) + %%table: + %rep %0 - 1 + dd %%base %+ .%2 - %%table + %rotate 1 + %endrep +%endmacro + +%macro CDEF_FILTER_JMP_TABLE 1 +JMP_TABLE cdef_filter_%1, \ + d6k0, d6k1, d7k0, d7k1, \ + d0k0, d0k1, d1k0, d1k1, d2k0, d2k1, d3k0, d3k1, \ + d4k0, d4k1, d5k0, d5k1, d6k0, d6k1, d7k0, d7k1, \ + d0k0, d0k1, d1k0, d1k1 +%endmacro + +SECTION_RODATA 32 + +pd_47130256: dd 4, 7, 1, 3, 0, 2, 5, 6 +blend_4x4: dd 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00 + dd 0x80, 0x00, 0x00 +blend_4x8_0: dd 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +blend_4x8_1: dd 0x00, 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + dd 0x00, 0x00 +blend_4x8_2: dd 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 + dd 0x0000 +blend_4x8_3: dd 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 + dd 0x0000, 0x0000 +blend_8x8_0: dq 0x00, 0x00, 0x80, 0x80, 0x80, 0x80 +blend_8x8_1: dq 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x0000, 0x0000 +div_table: dd 840, 420, 280, 210, 168, 140, 120, 105, 420, 210, 140, 105 +shufw_6543210x:db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 +shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 +pw_128: times 2 dw 128 +pw_2048: times 2 dw 2048 +tap_table: ; masks for 8 bit shifts + db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 + ; weights + db 4, 2, 3, 3, 2, 1 + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + db 1 * 16 + 0, 2 * 16 + 0 + db 1 * 16 + 0, 2 * 16 - 1 + ; the last 6 are repeats of the first 6 so we don't need to & 7 + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + +CDEF_FILTER_JMP_TABLE 4x4 +CDEF_FILTER_JMP_TABLE 4x8 +CDEF_FILTER_JMP_TABLE 8x8 + +SECTION .text + +%macro PREP_REGS 2 ; w, h + ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] + mov dird, r6m + lea tableq, [cdef_filter_%1x%2_jmptable] + lea dirq, [tableq+dirq*2*4] +%if %1 == 4 + %if %2 == 4 + DEFINE_ARGS dst, stride, left, top, pri, sec, \ + table, dir, dirjmp, dst4, stride3, k + %else + DEFINE_ARGS dst, stride, left, top, pri, sec, \ + table, dir, dirjmp, dst4, dst8, stride3, k + lea dst8q, [dstq+strideq*8] + %endif +%else + DEFINE_ARGS dst, stride, h, top1, pri, sec, \ + table, dir, dirjmp, top2, dst4, stride3, k + mov hq, -8 + lea top1q, [top1q+strideq*0] + lea top2q, [top1q+strideq*1] +%endif + lea dst4q, [dstq+strideq*4] +%if %1 == 4 + lea stride3q, [strideq*3] +%endif +%endmacro + +%macro LOAD_BLOCK 2-3 0 ; w, h, init_min_max + mov kd, 1 + pxor m15, m15 ; sum +%if %2 == 8 + pxor m12, m12 + %if %1 == 4 + movd xm4, [dstq +strideq*0] + movd xm6, [dstq +strideq*1] + movd xm5, [dstq +strideq*2] + movd xm7, [dstq +stride3q ] + vinserti128 m4, [dst4q+strideq*0], 1 + vinserti128 m6, [dst4q+strideq*1], 1 + vinserti128 m5, [dst4q+strideq*2], 1 + vinserti128 m7, [dst4q+stride3q ], 1 + punpckldq m4, m6 + punpckldq m5, m7 + %else + movq xm4, [dstq+strideq*0] + movq xm5, [dstq+strideq*1] + vinserti128 m4, [dstq+strideq*2], 1 + vinserti128 m5, [dstq+stride3q ], 1 + %endif + punpcklqdq m4, m5 +%else + movd xm4, [dstq+strideq*0] + movd xm5, [dstq+strideq*1] + vinserti128 m4, [dstq+strideq*2], 1 + vinserti128 m5, [dstq+stride3q ], 1 + punpckldq m4, m5 +%endif +%if %3 == 1 + mova m7, m4 ; min + mova m8, m4 ; max +%endif +%endmacro + +%macro ACCUMULATE_TAP_BYTE 7-8 0 ; tap_offset, shift, mask, strength + ; mul_tap, w, h, clip + ; load p0/p1 + movsxd dirjmpq, [dirq+kq*4+%1*2*4] + add dirjmpq, tableq + call dirjmpq + +%if %8 == 1 + pmaxub m7, m5 + pminub m8, m5 + pmaxub m7, m6 + pminub m8, m6 +%endif + + ; accumulate sum[m15] over p0/p1 +%if %7 == 4 + punpcklbw m5, m6 + punpcklbw m6, m4, m4 + psubusb m9, m5, m6 + psubusb m5, m6, m5 + por m9, m5 ; abs_diff_p01(p01 - px) + pcmpeqb m5, m9 + por m5, %5 + psignb m6, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + pminub m5, m9 + pmaddubsw m5, m6 + paddw m15, m5 +%else + psubusb m9, m5, m4 + psubusb m5, m4, m5 + psubusb m11, m6, m4 + psubusb m6, m4, m6 + por m9, m5 ; abs_diff_p0(p0 - px) + por m11, m6 ; abs_diff_p1(p1 - px) + pcmpeqb m5, m9 + pcmpeqb m6, m11 + punpckhbw m10, m9, m11 + punpcklbw m9, m11 + por m5, %5 + por m11, m6, %5 + punpckhbw m6, m5, m11 + punpcklbw m5, m11 + psignb m11, %5, m6 + psrlw m6, m10, %2 ; emulate 8-bit shift + pand m6, %3 + psubusb m6, %4, m6 + pminub m6, m10 + pmaddubsw m6, m11 + paddw m12, m6 + psignb m11, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + pminub m5, m9 + pmaddubsw m5, m11 + paddw m15, m5 +%endif +%endmacro + +%macro ADJUST_PIXEL 4-5 0 ; w, h, zero, pw_2048, clip +%if %2 == 4 + %if %5 == 1 + punpcklbw m4, %3 + %endif + pcmpgtw %3, m15 + paddw m15, %3 + pmulhrsw m15, %4 + %if %5 == 0 + packsswb m15, m15 + paddb m4, m15 + %else + paddw m4, m15 + packuswb m4, m4 ; clip px in [0x0,0xff] + pminub m4, m7 + pmaxub m4, m8 + %endif + vextracti128 xm5, m4, 1 + movd [dstq+strideq*0], xm4 + movd [dstq+strideq*2], xm5 + pextrd [dstq+strideq*1], xm4, 1 + pextrd [dstq+stride3q ], xm5, 1 +%else + pcmpgtw m6, %3, m12 + pcmpgtw m5, %3, m15 + paddw m12, m6 + paddw m15, m5 + %if %5 == 1 + punpckhbw m5, m4, %3 + punpcklbw m4, %3 + %endif + pmulhrsw m12, %4 + pmulhrsw m15, %4 + %if %5 == 0 + packsswb m15, m12 + paddb m4, m15 + %else + paddw m5, m12 + paddw m4, m15 + packuswb m4, m5 ; clip px in [0x0,0xff] + pminub m4, m7 + pmaxub m4, m8 + %endif + vextracti128 xm5, m4, 1 + %if %1 == 4 + movd [dstq +strideq*0], xm4 + movd [dst4q+strideq*0], xm5 + pextrd [dstq +strideq*1], xm4, 1 + pextrd [dst4q+strideq*1], xm5, 1 + pextrd [dstq +strideq*2], xm4, 2 + pextrd [dst4q+strideq*2], xm5, 2 + pextrd [dstq +stride3q ], xm4, 3 + pextrd [dst4q+stride3q ], xm5, 3 + %else + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*2], xm5 + movhps [dstq+strideq*1], xm4 + movhps [dstq+stride3q ], xm5 + %endif +%endif +%endmacro + +%macro BORDER_PREP_REGS 2 ; w, h + ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] + mov dird, r6m + lea dirq, [tableq+dirq*2+14] +%if %1*%2*2/mmsize > 1 + %if %1 == 4 + DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, h, off, k + %else + DEFINE_ARGS dst, stride, dir, stk, pri, sec, h, off, k + %endif + mov hd, %1*%2*2/mmsize +%else + DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, off, k +%endif + lea stkq, [px] + pxor m11, m11 +%endmacro + +%macro BORDER_LOAD_BLOCK 2-3 0 ; w, h, init_min_max + mov kd, 1 +%if %1 == 4 + movq xm4, [stkq+32*0] + movhps xm4, [stkq+32*1] + movq xm5, [stkq+32*2] + movhps xm5, [stkq+32*3] + vinserti128 m4, xm5, 1 +%else + mova xm4, [stkq+32*0] ; px + vinserti128 m4, [stkq+32*1], 1 +%endif + pxor m15, m15 ; sum +%if %3 == 1 + mova m7, m4 ; max + mova m8, m4 ; min +%endif +%endmacro + +%macro ACCUMULATE_TAP_WORD 6-7 0 ; tap_offset, shift, mask, strength + ; mul_tap, w, clip + ; load p0/p1 + movsx offq, byte [dirq+kq+%1] ; off1 +%if %6 == 4 + movq xm5, [stkq+offq*2+32*0] ; p0 + movq xm6, [stkq+offq*2+32*2] + movhps xm5, [stkq+offq*2+32*1] + movhps xm6, [stkq+offq*2+32*3] + vinserti128 m5, xm6, 1 +%else + movu xm5, [stkq+offq*2+32*0] ; p0 + vinserti128 m5, [stkq+offq*2+32*1], 1 +%endif + neg offq ; -off1 +%if %6 == 4 + movq xm6, [stkq+offq*2+32*0] ; p1 + movq xm9, [stkq+offq*2+32*2] + movhps xm6, [stkq+offq*2+32*1] + movhps xm9, [stkq+offq*2+32*3] + vinserti128 m6, xm9, 1 +%else + movu xm6, [stkq+offq*2+32*0] ; p1 + vinserti128 m6, [stkq+offq*2+32*1], 1 +%endif +%if %7 == 1 + ; out of bounds values are set to a value that is a both a large unsigned + ; value and a negative signed value. + ; use signed max and unsigned min to remove them + pmaxsw m7, m5 ; max after p0 + pminuw m8, m5 ; min after p0 + pmaxsw m7, m6 ; max after p1 + pminuw m8, m6 ; min after p1 +%endif + + ; accumulate sum[m15] over p0/p1 + ; calculate difference before converting + psubw m5, m4 ; diff_p0(p0 - px) + psubw m6, m4 ; diff_p1(p1 - px) + + ; convert to 8-bits with signed saturation + ; saturating to large diffs has no impact on the results + packsswb m5, m6 + + ; group into pairs so we can accumulate using maddubsw + pshufb m5, m12 + pabsb m9, m5 + psignb m10, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + + ; use unsigned min since abs diff can equal 0x80 + pminub m5, m9 + pmaddubsw m5, m10 + paddw m15, m5 +%endmacro + +%macro BORDER_ADJUST_PIXEL 2-3 0 ; w, pw_2048, clip + pcmpgtw m9, m11, m15 + paddw m15, m9 + pmulhrsw m15, %2 + paddw m4, m15 +%if %3 == 1 + pminsw m4, m7 + pmaxsw m4, m8 +%endif + packuswb m4, m4 + vextracti128 xm5, m4, 1 +%if %1 == 4 + movd [dstq+strideq*0], xm4 + pextrd [dstq+strideq*1], xm4, 1 + movd [dstq+strideq*2], xm5 + pextrd [dstq+stride3q], xm5, 1 +%else + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*1], xm5 +%endif +%endmacro + +%macro CDEF_FILTER 2 ; w, h +INIT_YMM avx2 +cglobal cdef_filter_%1x%2, 4, 9, 0, dst, stride, left, top, \ + pri, sec, dir, damping, edge +%assign stack_offset_entry stack_offset + mov edged, edgem + cmp edged, 0xf + jne .border_block + + PUSH r9 + PUSH r10 + PUSH r11 +%if %2 == 4 + %assign regs_used 12 + %if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 + %endif + ALLOC_STACK 0x60, 16 + pmovzxbw xm0, [leftq+1] + vpermq m0, m0, q0110 + psrldq m1, m0, 4 + vpalignr m2, m0, m0, 12 + movu [rsp+0x10], m0 + movu [rsp+0x28], m1 + movu [rsp+0x40], m2 +%elif %1 == 4 + PUSH r12 + %assign regs_used 13 + %if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 + %endif + ALLOC_STACK 8*2+%1*%2*1, 16 + pmovzxwd m0, [leftq] + mova [rsp+0x10], m0 +%else + PUSH r12 + PUSH r13 + %assign regs_used 14 + %if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 + %endif + ALLOC_STACK 8*2+%1*%2*2+32, 16 + lea r11, [strideq*3] + movu xm4, [dstq+strideq*2] + pmovzxwq m0, [leftq+0] + pmovzxwq m1, [leftq+8] + vinserti128 m4, [dstq+r11], 1 + pmovzxbd m2, [leftq+1] + pmovzxbd m3, [leftq+9] + mova [rsp+0x10], m0 + mova [rsp+0x30], m1 + mova [rsp+0x50], m2 + mova [rsp+0x70], m3 + mova [rsp+0x90], m4 +%endif + + DEFINE_ARGS dst, stride, left, top, pri, secdmp, zero, pridmp, damping + mov dampingd, r7m + xor zerod, zerod + movifnidn prid, prim + sub dampingd, 31 + movifnidn secdmpd, secdmpm + or prid, 0 + jz .sec_only + movd xm0, prid + lzcnt pridmpd, prid + add pridmpd, dampingd + cmovs pridmpd, zerod + mov [rsp+0], pridmpq ; pri_shift + or secdmpd, 0 + jz .pri_only + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + + DEFINE_ARGS dst, stride, left, top, pri, secdmp, table, pridmp + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, pri, sec, table, dir + vpbroadcastb m0, xm0 ; pri_strength + vpbroadcastb m1, xm1 ; sec_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + lea secq, [tableq+12] ; sec_taps + + PREP_REGS %1, %2 +%if %1*%2 > mmsize +.v_loop: +%endif + LOAD_BLOCK %1, %2, 1 +.k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2, 1 ; dir + 0 + ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir + 2 + ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir - 2 + dec kq + jge .k_loop + + vpbroadcastd m10, [pw_2048] + pxor m9, m9 + ADJUST_PIXEL %1, %2, m9, m10, 1 +%if %1*%2 > mmsize + mov dstq, dst4q + lea top1q, [rsp+0x90] + lea top2q, [rsp+0xA0] + lea dst4q, [dst4q+strideq*4] + add hq, 4 + jl .v_loop +%endif + RET + +.pri_only: + DEFINE_ARGS dst, stride, left, top, pri, _, table, pridmp + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, pri, _, table, dir + vpbroadcastb m0, xm0 ; pri_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + PREP_REGS %1, %2 + vpbroadcastd m3, [pw_2048] + pxor m1, m1 +%if %1*%2 > mmsize +.pri_v_loop: +%endif + LOAD_BLOCK %1, %2 +.pri_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2 ; dir + 0 + dec kq + jge .pri_k_loop + ADJUST_PIXEL %1, %2, m1, m3 +%if %1*%2 > mmsize + mov dstq, dst4q + lea top1q, [rsp+0x90] + lea top2q, [rsp+0xA0] + lea dst4q, [dst4q+strideq*4] + add hq, 4 + jl .pri_v_loop +%endif + RET + +.sec_only: + DEFINE_ARGS dst, stride, left, top, _, secdmp, zero, _, damping + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + DEFINE_ARGS dst, stride, left, top, _, secdmp, table + lea tableq, [tap_table] + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, _, sec, table, dir + vpbroadcastb m1, xm1 ; sec_strength + lea secq, [tableq+12] ; sec_taps + PREP_REGS %1, %2 + vpbroadcastd m2, [pw_2048] + pxor m0, m0 +%if %1*%2 > mmsize +.sec_v_loop: +%endif + LOAD_BLOCK %1, %2 +.sec_k_loop: + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2 ; dir + 2 + ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2 ; dir - 2 + dec kq + jge .sec_k_loop + ADJUST_PIXEL %1, %2, m0, m2 +%if %1*%2 > mmsize + mov dstq, dst4q + lea top1q, [rsp+0x90] + lea top2q, [rsp+0xA0] + lea dst4q, [dst4q+strideq*4] + add hq, 4 + jl .sec_v_loop +%endif + RET + +.d0k0: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m6, [dstq+strideq*1-1] + vpbroadcastq m10, [dstq+strideq*2-1] + movd xm5, [topq+strideq*1+1] + movd xm9, [dstq+strideq*0+1] + psrldq m11, m6, 2 + psrldq m12, m10, 2 + vinserti128 m6, [dstq+stride3q -1], 1 + vinserti128 m10, [dstq+strideq*4-1], 1 + vpblendd m5, m11, 0x10 + vpblendd m9, m12, 0x10 + movu m11, [blend_4x4+16] + punpckldq m6, m10 + punpckldq m5, m9 + vpblendvb m6, [rsp+gprsize+0x28], m11 + %else + movd xm5, [topq +strideq*1+1] + movq xm6, [dstq +strideq*1-1] + movq xm10, [dstq +stride3q -1] + movq xm11, [dst4q+strideq*1-1] + pinsrd xm5, [dstq +strideq*0+1], 1 + movhps xm6, [dstq +strideq*2-1] + movhps xm10, [dst4q+strideq*0-1] + movhps xm11, [dst4q+strideq*2-1] + psrldq xm9, xm6, 2 + shufps xm5, xm9, q2010 ; -1 +0 +1 +2 + shufps xm6, xm10, q2020 ; +1 +2 +3 +4 + psrldq xm9, xm11, 2 + psrldq xm10, 2 + shufps xm10, xm9, q2020 ; +3 +4 +5 +6 + movd xm9, [dst4q+stride3q -1] + pinsrd xm9, [dst4q+strideq*4-1], 1 + shufps xm11, xm9, q1020 ; +5 +6 +7 +8 + pmovzxbw m9, [leftq+3] + vinserti128 m6, xm11, 1 + movu m11, [blend_4x8_0+4] + vinserti128 m5, xm10, 1 + vpblendvb m6, m9, m11 + %endif +%else + lea r13, [blend_8x8_0+16] + movq xm5, [top2q +1] + vbroadcasti128 m10, [dstq+strideq*1-1] + vbroadcasti128 m11, [dstq+strideq*2-1] + movhps xm5, [dstq+strideq*0+1] + vinserti128 m6, m10, [dstq+stride3q -1], 1 + vinserti128 m9, m11, [dstq+strideq*4-1], 1 + psrldq m10, 2 + psrldq m11, 2 + punpcklqdq m6, m9 + movu m9, [r13+hq*2*1+16*1] + punpcklqdq m10, m11 + vpblendd m5, m10, 0xF0 + vpblendvb m6, [rsp+gprsize+80+hq*8+64+8*1], m9 +%endif + ret +.d1k0: +.d2k0: +.d3k0: +%if %1 == 4 + %if %2 == 4 + movq xm6, [dstq+strideq*0-1] + movq xm9, [dstq+strideq*1-1] + vinserti128 m6, [dstq+strideq*2-1], 1 + vinserti128 m9, [dstq+stride3q -1], 1 + movu m11, [rsp+gprsize+0x10] + pcmpeqd m12, m12 + psrldq m5, m6, 2 + psrldq m10, m9, 2 + psrld m12, 24 + punpckldq m6, m9 + punpckldq m5, m10 + vpblendvb m6, m11, m12 + %else + movq xm6, [dstq +strideq*0-1] + movq xm9, [dstq +strideq*2-1] + movhps xm6, [dstq +strideq*1-1] + movhps xm9, [dstq +stride3q -1] + movq xm10, [dst4q+strideq*0-1] + movhps xm10, [dst4q+strideq*1-1] + psrldq xm5, xm6, 2 + psrldq xm11, xm9, 2 + shufps xm5, xm11, q2020 + movq xm11, [dst4q+strideq*2-1] + movhps xm11, [dst4q+stride3q -1] + shufps xm6, xm9, q2020 + shufps xm9, xm10, xm11, q2020 + vinserti128 m6, xm9, 1 + pmovzxbw m9, [leftq+1] + psrldq xm10, 2 + psrldq xm11, 2 + shufps xm10, xm11, q2020 + vpbroadcastd m11, [blend_4x8_0+4] + vinserti128 m5, xm10, 1 + vpblendvb m6, m9, m11 + %endif +%else + movu xm5, [dstq+strideq*0-1] + movu xm9, [dstq+strideq*1-1] + vinserti128 m5, [dstq+strideq*2-1], 1 + vinserti128 m9, [dstq+stride3q -1], 1 + movu m10, [blend_8x8_0+16] + punpcklqdq m6, m5, m9 + vpblendvb m6, [rsp+gprsize+80+hq*8+64], m10 + psrldq m5, 2 + psrldq m9, 2 + punpcklqdq m5, m9 +%endif + ret +.d4k0: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m10, [dstq+strideq*1-1] + vpbroadcastq m11, [dstq+strideq*2-1] + movd xm6, [topq+strideq*1-1] + movd xm9, [dstq+strideq*0-1] + psrldq m5, m10, 2 + psrldq m12, m11, 2 + vpblendd m6, m10, 0x10 + vpblendd m9, m11, 0x10 + movu m10, [blend_4x4] + vinserti128 m5, [dstq+stride3q +1], 1 + vinserti128 m12, [dstq+strideq*4+1], 1 + punpckldq m6, m9 + punpckldq m5, m12 + vpblendvb m6, [rsp+gprsize+0x40], m10 + %else + movd xm6, [topq +strideq*1-1] + movq xm9, [dstq +strideq*1-1] + movq xm10, [dstq +stride3q -1] + movq xm11, [dst4q+strideq*1-1] + pinsrd xm6, [dstq +strideq*0-1], 1 + movhps xm9, [dstq +strideq*2-1] + movhps xm10, [dst4q+strideq*0-1] + movhps xm11, [dst4q+strideq*2-1] + psrldq xm5, xm9, 2 + shufps xm6, xm9, q2010 + psrldq xm9, xm10, 2 + shufps xm5, xm9, q2020 + shufps xm10, xm11, q2020 + movd xm9, [dst4q+stride3q +1] + vinserti128 m6, xm10, 1 + pinsrd xm9, [dst4q+strideq*4+1], 1 + psrldq xm11, 2 + pmovzxbw m10, [leftq-1] + shufps xm11, xm9, q1020 + movu m9, [blend_4x8_0] + vinserti128 m5, xm11, 1 + vpblendvb m6, m10, m9 + %endif +%else + lea r13, [blend_8x8_0+8] + movq xm6, [top2q -1] + vbroadcasti128 m5, [dstq+strideq*1-1] + vbroadcasti128 m9, [dstq+strideq*2-1] + movhps xm6, [dstq+strideq*0-1] + movu m11, [r13+hq*2*1+16*1] + punpcklqdq m10, m5, m9 + vinserti128 m5, [dstq+stride3q -1], 1 + vinserti128 m9, [dstq+strideq*4-1], 1 + vpblendd m6, m10, 0xF0 + vpblendvb m6, [rsp+gprsize+80+hq*8+64-8*1], m11 + psrldq m5, 2 + psrldq m9, 2 + punpcklqdq m5, m9 +%endif + ret +.d5k0: +.d6k0: +.d7k0: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq+strideq*1 ] + vpbroadcastd m5, [dstq+strideq*1 ] + vpbroadcastd m9, [dstq+strideq*2 ] + vpblendd xm6, [dstq+strideq*0-4], 0x2 + vpblendd m5, m9, 0x22 + vpblendd m6, m5, 0x30 + vinserti128 m5, [dstq+stride3q ], 1 + vpblendd m5, [dstq+strideq*4-20], 0x20 + %else + movd xm6, [topq +strideq*1] + movd xm5, [dstq +strideq*1] + movd xm9, [dstq +stride3q ] + movd xm10, [dst4q+strideq*1] + movd xm11, [dst4q+stride3q ] + pinsrd xm6, [dstq +strideq*0], 1 + pinsrd xm5, [dstq +strideq*2], 1 + pinsrd xm9, [dst4q+strideq*0], 1 + pinsrd xm10, [dst4q+strideq*2], 1 + pinsrd xm11, [dst4q+strideq*4], 1 + punpcklqdq xm6, xm5 + punpcklqdq xm5, xm9 + punpcklqdq xm9, xm10 + punpcklqdq xm10, xm11 + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + %endif +%else + movq xm6, [top2q ] + movq xm5, [dstq+strideq*1] + movq xm9, [dstq+stride3q ] + movhps xm6, [dstq+strideq*0] + movhps xm5, [dstq+strideq*2] + movhps xm9, [dstq+strideq*4] + vinserti128 m6, xm5, 1 + vinserti128 m5, xm9, 1 +%endif + ret +.d0k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [dstq +strideq*2-2] + movd xm9, [dstq +stride3q -2] + movd xm5, [topq +strideq*0+2] + movd xm10, [topq +strideq*1+2] + pinsrw xm6, [leftq+4], 0 + pinsrw xm9, [leftq+6], 0 + vinserti128 m5, [dstq +strideq*0+2], 1 + vinserti128 m10, [dstq +strideq*1+2], 1 + vinserti128 m6, [dst4q+strideq*0-2], 1 + vinserti128 m9, [dst4q+strideq*1-2], 1 + punpckldq m5, m10 + punpckldq m6, m9 + %else + movq xm6, [dstq +strideq*2-2] + movd xm10, [dst4q+strideq*2-2] + movd xm5, [topq +strideq*0+2] + movq xm9, [dst4q+strideq*0-2] + movhps xm6, [dstq +stride3q -2] + pinsrw xm10, [dst4q+stride3q ], 3 + pinsrd xm5, [topq +strideq*1+2], 1 + movhps xm9, [dst4q+strideq*1-2] + pinsrd xm10, [dst8q+strideq*0-2], 2 + pinsrd xm5, [dstq +strideq*0+2], 2 + pinsrd xm10, [dst8q+strideq*1-2], 3 + pinsrd xm5, [dstq +strideq*1+2], 3 + shufps xm11, xm6, xm9, q3131 + shufps xm6, xm9, q2020 + movu m9, [blend_4x8_3+8] + vinserti128 m6, xm10, 1 + vinserti128 m5, xm11, 1 + vpblendvb m6, [rsp+gprsize+16+8], m9 + %endif +%else + lea r13, [blend_8x8_1+16] + movq xm6, [dstq +strideq*2-2] + movq xm9, [dstq +stride3q -2] + movq xm5, [top1q +2] + movq xm10, [top2q +2] + movu m11, [r13+hq*2*2+16*2] + vinserti128 m6, [dst4q+strideq*0-2], 1 + vinserti128 m9, [dst4q+strideq*1-2], 1 + vinserti128 m5, [dstq +strideq*0+2], 1 + vinserti128 m10, [dstq +strideq*1+2], 1 + punpcklqdq m6, m9 + punpcklqdq m5, m10 + vpblendvb m6, [rsp+gprsize+16+hq*8+64+8*2], m11 +%endif + ret +.d1k1: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m6, [dstq+strideq*1-2] + vpbroadcastq m9, [dstq+strideq*2-2] + movd xm5, [topq+strideq*1+2] + movd xm10, [dstq+strideq*0+2] + psrldq m11, m6, 4 + psrldq m12, m9, 4 + vpblendd m5, m11, 0x10 + movq xm11, [leftq+2] + vinserti128 m6, [dstq+stride3q -2], 1 + punpckldq xm11, xm11 + vpblendd m10, m12, 0x10 + pcmpeqd m12, m12 + pmovzxwd m11, xm11 + psrld m12, 16 + punpckldq m6, m9 + vpbroadcastd m9, [dstq+strideq*4-2] + vpblendvb m6, m11, m12 + punpckldq m5, m10 + vpblendd m6, m9, 0x20 + %else + movd xm5, [topq +strideq*1+2] + movq xm6, [dstq +strideq*1-2] + movq xm9, [dstq +stride3q -2] + movq xm10, [dst4q+strideq*1-2] + movd xm11, [dst4q+stride3q -2] + pinsrd xm5, [dstq +strideq*0+2], 1 + movhps xm6, [dstq +strideq*2-2] + movhps xm9, [dst4q+strideq*0-2] + movhps xm10, [dst4q+strideq*2-2] + pinsrd xm11, [dst4q+strideq*4-2], 1 + shufps xm5, xm6, q3110 + shufps xm6, xm9, q2020 + shufps xm9, xm10, q3131 + shufps xm10, xm11, q1020 + movu m11, [blend_4x8_2+4] + vinserti128 m6, xm10, 1 + vinserti128 m5, xm9, 1 + vpblendvb m6, [rsp+gprsize+16+4], m11 + %endif +%else + lea r13, [blend_8x8_1+16] + movq xm5, [top2q +2] + vbroadcasti128 m6, [dstq+strideq*1-2] + vbroadcasti128 m9, [dstq+strideq*2-2] + movhps xm5, [dstq+strideq*0+2] + shufps m10, m6, m9, q2121 + vinserti128 m6, [dstq+stride3q -2], 1 + vinserti128 m9, [dstq+strideq*4-2], 1 + movu m11, [r13+hq*2*1+16*1] + vpblendd m5, m10, 0xF0 + punpcklqdq m6, m9 + vpblendvb m6, [rsp+gprsize+16+hq*8+64+8*1], m11 +%endif + ret +.d2k1: +%if %1 == 4 + %if %2 == 4 + movq xm11, [leftq] + movq xm6, [dstq+strideq*0-2] + movq xm9, [dstq+strideq*1-2] + vinserti128 m6, [dstq+strideq*2-2], 1 + vinserti128 m9, [dstq+stride3q -2], 1 + punpckldq xm11, xm11 + psrldq m5, m6, 4 + psrldq m10, m9, 4 + pmovzxwd m11, xm11 + punpckldq m6, m9 + punpckldq m5, m10 + pblendw m6, m11, 0x05 + %else + movq xm5, [dstq +strideq*0-2] + movq xm9, [dstq +strideq*2-2] + movq xm10, [dst4q+strideq*0-2] + movq xm11, [dst4q+strideq*2-2] + movhps xm5, [dstq +strideq*1-2] + movhps xm9, [dstq +stride3q -2] + movhps xm10, [dst4q+strideq*1-2] + movhps xm11, [dst4q+stride3q -2] + shufps xm6, xm5, xm9, q2020 + shufps xm5, xm9, q3131 + shufps xm9, xm10, xm11, q2020 + shufps xm10, xm11, q3131 + pmovzxwd m11, [leftq] + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + pblendw m6, m11, 0x55 + %endif +%else + mova m11, [rsp+gprsize+16+hq*8+64] + movu xm5, [dstq+strideq*0-2] + movu xm9, [dstq+strideq*1-2] + vinserti128 m5, [dstq+strideq*2-2], 1 + vinserti128 m9, [dstq+stride3q -2], 1 + shufps m6, m5, m9, q1010 + shufps m5, m9, q2121 + pblendw m6, m11, 0x11 +%endif + ret +.d3k1: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m11, [dstq+strideq*1-2] + vpbroadcastq m12, [dstq+strideq*2-2] + movd xm6, [topq+strideq*1-2] + movd xm9, [dstq+strideq*0-2] + pblendw m11, [leftq-16+2], 0x01 + pblendw m12, [leftq-16+4], 0x01 + pinsrw xm9, [leftq- 0+0], 0 + psrldq m5, m11, 4 + psrldq m10, m12, 4 + vinserti128 m5, [dstq+stride3q +2], 1 + vinserti128 m10, [dstq+strideq*4+2], 1 + vpblendd m6, m11, 0x10 + vpblendd m9, m12, 0x10 + punpckldq m6, m9 + punpckldq m5, m10 + %else + movd xm6, [topq +strideq*1-2] + movq xm5, [dstq +strideq*1-2] + movq xm9, [dstq +stride3q -2] + movq xm10, [dst4q+strideq*1-2] + movd xm11, [dst4q+stride3q +2] + pinsrw xm6, [dstq +strideq*0 ], 3 + movhps xm5, [dstq +strideq*2-2] + movhps xm9, [dst4q+strideq*0-2] + movhps xm10, [dst4q+strideq*2-2] + pinsrd xm11, [dst4q+strideq*4+2], 1 + shufps xm6, xm5, q2010 + shufps xm5, xm9, q3131 + shufps xm9, xm10, q2020 + shufps xm10, xm11, q1031 + movu m11, [blend_4x8_2] + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + vpblendvb m6, [rsp+gprsize+16-4], m11 + %endif +%else + lea r13, [blend_8x8_1+8] + movq xm6, [top2q -2] + vbroadcasti128 m5, [dstq+strideq*1-2] + vbroadcasti128 m10, [dstq+strideq*2-2] + movhps xm6, [dstq+strideq*0-2] + punpcklqdq m9, m5, m10 + vinserti128 m5, [dstq+stride3q -2], 1 + vinserti128 m10, [dstq+strideq*4-2], 1 + movu m11, [r13+hq*2*1+16*1] + vpblendd m6, m9, 0xF0 + shufps m5, m10, q2121 + vpblendvb m6, [rsp+gprsize+16+hq*8+64-8*1], m11 +%endif + ret +.d4k1: +%if %1 == 4 + %if %2 == 4 + vinserti128 m6, [dstq +strideq*0-2], 1 + vinserti128 m9, [dstq +strideq*1-2], 1 + movd xm5, [dstq +strideq*2+2] + movd xm10, [dstq +stride3q +2] + pblendw m6, [leftq-16+0], 0x01 + pblendw m9, [leftq-16+2], 0x01 + vinserti128 m5, [dst4q+strideq*0+2], 1 + vinserti128 m10, [dst4q+strideq*1+2], 1 + vpblendd m6, [topq +strideq*0-2], 0x01 + vpblendd m9, [topq +strideq*1-2], 0x01 + punpckldq m5, m10 + punpckldq m6, m9 + %else + movd xm6, [topq +strideq*0-2] + movq xm5, [dstq +strideq*2-2] + movq xm9, [dst4q+strideq*0-2] + movd xm10, [dst4q+strideq*2+2] + pinsrd xm6, [topq +strideq*1-2], 1 + movhps xm5, [dstq +stride3q -2] + movhps xm9, [dst4q+strideq*1-2] + pinsrd xm10, [dst4q+stride3q +2], 1 + pinsrd xm6, [dstq +strideq*0-2], 2 + pinsrd xm10, [dst8q+strideq*0+2], 2 + pinsrd xm6, [dstq +strideq*1-2], 3 + pinsrd xm10, [dst8q+strideq*1+2], 3 + shufps xm11, xm5, xm9, q2020 + shufps xm5, xm9, q3131 + movu m9, [blend_4x8_3] + vinserti128 m6, xm11, 1 + vinserti128 m5, xm10, 1 + vpblendvb m6, [rsp+gprsize+16-8], m9 + %endif +%else + lea r13, [blend_8x8_1] + movu m11, [r13+hq*2*2+16*2] + movq xm6, [top1q -2] + movq xm9, [top2q -2] + movq xm5, [dstq +strideq*2+2] + movq xm10, [dstq +stride3q +2] + vinserti128 m6, [dstq +strideq*0-2], 1 + vinserti128 m9, [dstq +strideq*1-2], 1 + vinserti128 m5, [dst4q+strideq*0+2], 1 + vinserti128 m10, [dst4q+strideq*1+2], 1 + punpcklqdq m6, m9 + vpblendvb m6, [rsp+gprsize+16+hq*8+64-8*2], m11 + punpcklqdq m5, m10 +%endif + ret +.d5k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq +strideq*0-1] + movd xm9, [topq +strideq*1-1] + movd xm5, [dstq +strideq*2+1] + movd xm10, [dstq +stride3q +1] + pcmpeqd m12, m12 + pmovzxbw m11, [leftq-8+1] + psrld m12, 24 + vinserti128 m6, [dstq +strideq*0-1], 1 + vinserti128 m9, [dstq +strideq*1-1], 1 + vinserti128 m5, [dst4q+strideq*0+1], 1 + vinserti128 m10, [dst4q+strideq*1+1], 1 + punpckldq m6, m9 + pxor m9, m9 + vpblendd m12, m9, 0x0F + punpckldq m5, m10 + vpblendvb m6, m11, m12 + %else + movd xm6, [topq +strideq*0-1] + movq xm5, [dstq +strideq*2-1] + movq xm9, [dst4q+strideq*0-1] + movd xm10, [dst4q+strideq*2+1] + pinsrd xm6, [topq +strideq*1-1], 1 + movhps xm5, [dstq +stride3q -1] + movhps xm9, [dst4q+strideq*1-1] + pinsrd xm10, [dst4q+stride3q +1], 1 + pinsrd xm6, [dstq +strideq*0-1], 2 + pinsrd xm10, [dst8q+strideq*0+1], 2 + pinsrd xm6, [dstq +strideq*1-1], 3 + pinsrd xm10, [dst8q+strideq*1+1], 3 + shufps xm11, xm5, xm9, q2020 + vinserti128 m6, xm11, 1 + pmovzxbw m11, [leftq-3] + psrldq xm5, 2 + psrldq xm9, 2 + shufps xm5, xm9, q2020 + movu m9, [blend_4x8_1] + vinserti128 m5, xm10, 1 + vpblendvb m6, m11, m9 + %endif +%else + lea r13, [blend_8x8_0] + movu m11, [r13+hq*2*2+16*2] + movq xm6, [top1q -1] + movq xm9, [top2q -1] + movq xm5, [dstq +strideq*2+1] + movq xm10, [dstq +stride3q +1] + vinserti128 m6, [dstq +strideq*0-1], 1 + vinserti128 m9, [dstq +strideq*1-1], 1 + vinserti128 m5, [dst4q+strideq*0+1], 1 + vinserti128 m10, [dst4q+strideq*1+1], 1 + punpcklqdq m6, m9 + punpcklqdq m5, m10 + vpblendvb m6, [rsp+gprsize+80+hq*8+64-8*2], m11 +%endif + ret +.d6k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq +strideq*0] + movd xm9, [topq +strideq*1] + movd xm5, [dstq +strideq*2] + movd xm10, [dstq +stride3q ] + vinserti128 m6, [dstq +strideq*0], 1 + vinserti128 m9, [dstq +strideq*1], 1 + vinserti128 m5, [dst4q+strideq*0], 1 + vinserti128 m10, [dst4q+strideq*1], 1 + punpckldq m6, m9 + punpckldq m5, m10 + %else + movd xm5, [dstq +strideq*2] + movd xm6, [topq +strideq*0] + movd xm9, [dst4q+strideq*2] + pinsrd xm5, [dstq +stride3q ], 1 + pinsrd xm6, [topq +strideq*1], 1 + pinsrd xm9, [dst4q+stride3q ], 1 + pinsrd xm5, [dst4q+strideq*0], 2 + pinsrd xm6, [dstq +strideq*0], 2 + pinsrd xm9, [dst8q+strideq*0], 2 + pinsrd xm5, [dst4q+strideq*1], 3 + pinsrd xm6, [dstq +strideq*1], 3 + pinsrd xm9, [dst8q+strideq*1], 3 + vinserti128 m6, xm5, 1 + vinserti128 m5, xm9, 1 + %endif +%else + movq xm5, [dstq +strideq*2] + movq xm9, [dst4q+strideq*0] + movq xm6, [top1q ] + movq xm10, [dstq +strideq*0] + movhps xm5, [dstq +stride3q ] + movhps xm9, [dst4q+strideq*1] + movhps xm6, [top2q ] + movhps xm10, [dstq +strideq*1] + vinserti128 m5, xm9, 1 + vinserti128 m6, xm10, 1 +%endif + ret +.d7k1: +%if %1 == 4 + %if %2 == 4 + movd xm5, [dstq +strideq*2-1] + movd xm9, [dstq +stride3q -1] + movd xm6, [topq +strideq*0+1] + movd xm10, [topq +strideq*1+1] + pinsrb xm5, [leftq+ 5], 0 + pinsrb xm9, [leftq+ 7], 0 + vinserti128 m6, [dstq +strideq*0+1], 1 + vinserti128 m10, [dstq +strideq*1+1], 1 + vinserti128 m5, [dst4q+strideq*0-1], 1 + vinserti128 m9, [dst4q+strideq*1-1], 1 + punpckldq m6, m10 + punpckldq m5, m9 + %else + movd xm6, [topq +strideq*0+1] + movq xm9, [dstq +strideq*2-1] + movq xm10, [dst4q+strideq*0-1] + movd xm11, [dst4q+strideq*2-1] + pinsrd xm6, [topq +strideq*1+1], 1 + movhps xm9, [dstq +stride3q -1] + movhps xm10, [dst4q+strideq*1-1] + pinsrd xm11, [dst4q+stride3q -1], 1 + pinsrd xm6, [dstq +strideq*0+1], 2 + pinsrd xm11, [dst8q+strideq*0-1], 2 + pinsrd xm6, [dstq +strideq*1+1], 3 + pinsrd xm11, [dst8q+strideq*1-1], 3 + shufps xm5, xm9, xm10, q2020 + vinserti128 m5, xm11, 1 + pmovzxbw m11, [leftq+5] + psrldq xm9, 2 + psrldq xm10, 2 + shufps xm9, xm10, q2020 + movu m10, [blend_4x8_1+8] + vinserti128 m6, xm9, 1 + vpblendvb m5, m11, m10 + %endif +%else + lea r13, [blend_8x8_0+16] + movq xm5, [dstq +strideq*2-1] + movq xm9, [dst4q+strideq*0-1] + movq xm6, [top1q +1] + movq xm10, [dstq +strideq*0+1] + movhps xm5, [dstq +stride3q -1] + movhps xm9, [dst4q+strideq*1-1] + movhps xm6, [top2q +1] + movhps xm10, [dstq +strideq*1+1] + movu m11, [r13+hq*2*2+16*2] + vinserti128 m5, xm9, 1 + vinserti128 m6, xm10, 1 + vpblendvb m5, [rsp+gprsize+80+hq*8+64+8*2], m11 +%endif + ret + +.border_block: + DEFINE_ARGS dst, stride, left, top, pri, sec, stride3, dst4, edge +%define rstk rsp +%assign stack_offset stack_offset_entry +%if %1 == 4 && %2 == 8 + PUSH r9 + %assign regs_used 10 +%else + %assign regs_used 9 +%endif +%if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 +%endif + ALLOC_STACK 2*16+(%2+4)*32, 16 +%define px rsp+2*16+2*32 + + pcmpeqw m14, m14 + psllw m14, 15 ; 0x8000 + + ; prepare pixel buffers - body/right +%if %1 == 4 + INIT_XMM avx2 +%endif +%if %2 == 8 + lea dst4q, [dstq+strideq*4] +%endif + lea stride3q, [strideq*3] + test edgeb, 2 ; have_right + jz .no_right + pmovzxbw m1, [dstq+strideq*0] + pmovzxbw m2, [dstq+strideq*1] + pmovzxbw m3, [dstq+strideq*2] + pmovzxbw m4, [dstq+stride3q] + mova [px+0*32], m1 + mova [px+1*32], m2 + mova [px+2*32], m3 + mova [px+3*32], m4 +%if %2 == 8 + pmovzxbw m1, [dst4q+strideq*0] + pmovzxbw m2, [dst4q+strideq*1] + pmovzxbw m3, [dst4q+strideq*2] + pmovzxbw m4, [dst4q+stride3q] + mova [px+4*32], m1 + mova [px+5*32], m2 + mova [px+6*32], m3 + mova [px+7*32], m4 +%endif + jmp .body_done +.no_right: +%if %1 == 4 + movd xm1, [dstq+strideq*0] + movd xm2, [dstq+strideq*1] + movd xm3, [dstq+strideq*2] + movd xm4, [dstq+stride3q] + pmovzxbw xm1, xm1 + pmovzxbw xm2, xm2 + pmovzxbw xm3, xm3 + pmovzxbw xm4, xm4 + movq [px+0*32], xm1 + movq [px+1*32], xm2 + movq [px+2*32], xm3 + movq [px+3*32], xm4 +%else + pmovzxbw xm1, [dstq+strideq*0] + pmovzxbw xm2, [dstq+strideq*1] + pmovzxbw xm3, [dstq+strideq*2] + pmovzxbw xm4, [dstq+stride3q] + mova [px+0*32], xm1 + mova [px+1*32], xm2 + mova [px+2*32], xm3 + mova [px+3*32], xm4 +%endif + movd [px+0*32+%1*2], xm14 + movd [px+1*32+%1*2], xm14 + movd [px+2*32+%1*2], xm14 + movd [px+3*32+%1*2], xm14 +%if %2 == 8 + %if %1 == 4 + movd xm1, [dst4q+strideq*0] + movd xm2, [dst4q+strideq*1] + movd xm3, [dst4q+strideq*2] + movd xm4, [dst4q+stride3q] + pmovzxbw xm1, xm1 + pmovzxbw xm2, xm2 + pmovzxbw xm3, xm3 + pmovzxbw xm4, xm4 + movq [px+4*32], xm1 + movq [px+5*32], xm2 + movq [px+6*32], xm3 + movq [px+7*32], xm4 + %else + pmovzxbw xm1, [dst4q+strideq*0] + pmovzxbw xm2, [dst4q+strideq*1] + pmovzxbw xm3, [dst4q+strideq*2] + pmovzxbw xm4, [dst4q+stride3q] + mova [px+4*32], xm1 + mova [px+5*32], xm2 + mova [px+6*32], xm3 + mova [px+7*32], xm4 + %endif + movd [px+4*32+%1*2], xm14 + movd [px+5*32+%1*2], xm14 + movd [px+6*32+%1*2], xm14 + movd [px+7*32+%1*2], xm14 +%endif +.body_done: + + ; top + test edgeb, 4 ; have_top + jz .no_top + test edgeb, 1 ; have_left + jz .top_no_left + test edgeb, 2 ; have_right + jz .top_no_right + pmovzxbw m1, [topq+strideq*0-(%1/2)] + pmovzxbw m2, [topq+strideq*1-(%1/2)] + movu [px-2*32-%1], m1 + movu [px-1*32-%1], m2 + jmp .top_done +.top_no_right: + pmovzxbw m1, [topq+strideq*0-%1] + pmovzxbw m2, [topq+strideq*1-%1] + movu [px-2*32-%1*2], m1 + movu [px-1*32-%1*2], m2 + movd [px-2*32+%1*2], xm14 + movd [px-1*32+%1*2], xm14 + jmp .top_done +.top_no_left: + test edgeb, 2 ; have_right + jz .top_no_left_right + pmovzxbw m1, [topq+strideq*0] + pmovzxbw m2, [topq+strideq*1] + mova [px-2*32+0], m1 + mova [px-1*32+0], m2 + movd [px-2*32-4], xm14 + movd [px-1*32-4], xm14 + jmp .top_done +.top_no_left_right: +%if %1 == 4 + movd xm1, [topq+strideq*0] + pinsrd xm1, [topq+strideq*1], 1 + pmovzxbw xm1, xm1 + movq [px-2*32+0], xm1 + movhps [px-1*32+0], xm1 +%else + pmovzxbw xm1, [topq+strideq*0] + pmovzxbw xm2, [topq+strideq*1] + mova [px-2*32+0], xm1 + mova [px-1*32+0], xm2 +%endif + movd [px-2*32-4], xm14 + movd [px-1*32-4], xm14 + movd [px-2*32+%1*2], xm14 + movd [px-1*32+%1*2], xm14 + jmp .top_done +.no_top: + movu [px-2*32-%1], m14 + movu [px-1*32-%1], m14 +.top_done: + + ; left + test edgeb, 1 ; have_left + jz .no_left + pmovzxbw xm1, [leftq+ 0] +%if %2 == 8 + pmovzxbw xm2, [leftq+ 8] +%endif + movd [px+0*32-4], xm1 + pextrd [px+1*32-4], xm1, 1 + pextrd [px+2*32-4], xm1, 2 + pextrd [px+3*32-4], xm1, 3 +%if %2 == 8 + movd [px+4*32-4], xm2 + pextrd [px+5*32-4], xm2, 1 + pextrd [px+6*32-4], xm2, 2 + pextrd [px+7*32-4], xm2, 3 +%endif + jmp .left_done +.no_left: + movd [px+0*32-4], xm14 + movd [px+1*32-4], xm14 + movd [px+2*32-4], xm14 + movd [px+3*32-4], xm14 +%if %2 == 8 + movd [px+4*32-4], xm14 + movd [px+5*32-4], xm14 + movd [px+6*32-4], xm14 + movd [px+7*32-4], xm14 +%endif +.left_done: + + ; bottom + DEFINE_ARGS dst, stride, dst8, dummy1, pri, sec, stride3, dummy3, edge + test edgeb, 8 ; have_bottom + jz .no_bottom + lea dst8q, [dstq+%2*strideq] + test edgeb, 1 ; have_left + jz .bottom_no_left + test edgeb, 2 ; have_right + jz .bottom_no_right + pmovzxbw m1, [dst8q-(%1/2)] + pmovzxbw m2, [dst8q+strideq-(%1/2)] + movu [px+(%2+0)*32-%1], m1 + movu [px+(%2+1)*32-%1], m2 + jmp .bottom_done +.bottom_no_right: + pmovzxbw m1, [dst8q-%1] + pmovzxbw m2, [dst8q+strideq-%1] + movu [px+(%2+0)*32-%1*2], m1 + movu [px+(%2+1)*32-%1*2], m2 +%if %1 == 8 + movd [px+(%2-1)*32+%1*2], xm14 ; overwritten by previous movu +%endif + movd [px+(%2+0)*32+%1*2], xm14 + movd [px+(%2+1)*32+%1*2], xm14 + jmp .bottom_done +.bottom_no_left: + test edgeb, 2 ; have_right + jz .bottom_no_left_right + pmovzxbw m1, [dst8q] + pmovzxbw m2, [dst8q+strideq] + mova [px+(%2+0)*32+0], m1 + mova [px+(%2+1)*32+0], m2 + movd [px+(%2+0)*32-4], xm14 + movd [px+(%2+1)*32-4], xm14 + jmp .bottom_done +.bottom_no_left_right: +%if %1 == 4 + movd xm1, [dst8q] + pinsrd xm1, [dst8q+strideq], 1 + pmovzxbw xm1, xm1 + movq [px+(%2+0)*32+0], xm1 + movhps [px+(%2+1)*32+0], xm1 +%else + pmovzxbw xm1, [dst8q] + pmovzxbw xm2, [dst8q+strideq] + mova [px+(%2+0)*32+0], xm1 + mova [px+(%2+1)*32+0], xm2 +%endif + movd [px+(%2+0)*32-4], xm14 + movd [px+(%2+1)*32-4], xm14 + movd [px+(%2+0)*32+%1*2], xm14 + movd [px+(%2+1)*32+%1*2], xm14 + jmp .bottom_done +.no_bottom: + movu [px+(%2+0)*32-%1], m14 + movu [px+(%2+1)*32-%1], m14 +.bottom_done: + + ; actual filter + INIT_YMM avx2 + DEFINE_ARGS dst, stride, pridmp, damping, pri, secdmp, stride3, zero +%undef edged + ; register to shuffle values into after packing + vbroadcasti128 m12, [shufb_lohi] + + mov dampingd, r7m + xor zerod, zerod + movifnidn prid, prim + sub dampingd, 31 + movifnidn secdmpd, secdmpm + or prid, 0 + jz .border_sec_only + movd xm0, prid + lzcnt pridmpd, prid + add pridmpd, dampingd + cmovs pridmpd, zerod + mov [rsp+0], pridmpq ; pri_shift + or secdmpd, 0 + jz .border_pri_only + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + + DEFINE_ARGS dst, stride, pridmp, table, pri, secdmp, stride3 + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, dir, table, pri, sec, stride3 + vpbroadcastb m0, xm0 ; pri_strength + vpbroadcastb m1, xm1 ; sec_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + lea secq, [tableq+12] ; sec_taps + + BORDER_PREP_REGS %1, %2 +%if %1*%2*2/mmsize > 1 +.border_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2, 1 +.border_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1, 1 + ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1, 1 + ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1, 1 + dec kq + jge .border_k_loop + + vpbroadcastd m10, [pw_2048] + BORDER_ADJUST_PIXEL %1, m10, 1 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_v_loop +%endif + RET + +.border_pri_only: + DEFINE_ARGS dst, stride, pridmp, table, pri, _, stride3 + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + DEFINE_ARGS dst, stride, dir, table, pri, _, stride3 + vpbroadcastb m0, xm0 ; pri_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + BORDER_PREP_REGS %1, %2 + vpbroadcastd m1, [pw_2048] +%if %1*%2*2/mmsize > 1 +.border_pri_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2 +.border_pri_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1 + dec kq + jge .border_pri_k_loop + BORDER_ADJUST_PIXEL %1, m1 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_pri_v_loop +%endif + RET + +.border_sec_only: + DEFINE_ARGS dst, stride, _, damping, _, secdmp, stride3, zero + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + DEFINE_ARGS dst, stride, _, table, _, secdmp, stride3 + lea tableq, [tap_table] + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + DEFINE_ARGS dst, stride, dir, table, _, sec, stride3 + vpbroadcastb m1, xm1 ; sec_strength + lea secq, [tableq+12] ; sec_taps + BORDER_PREP_REGS %1, %2 + vpbroadcastd m0, [pw_2048] +%if %1*%2*2/mmsize > 1 +.border_sec_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2 +.border_sec_k_loop: + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1 + ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1 + dec kq + jge .border_sec_k_loop + BORDER_ADJUST_PIXEL %1, m0 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_sec_v_loop +%endif + RET +%endmacro + +CDEF_FILTER 8, 8 +CDEF_FILTER 4, 8 +CDEF_FILTER 4, 4 + +INIT_YMM avx2 +cglobal cdef_dir, 3, 4, 15, src, stride, var, stride3 + lea stride3q, [strideq*3] + movq xm0, [srcq+strideq*0] + movq xm1, [srcq+strideq*1] + movq xm2, [srcq+strideq*2] + movq xm3, [srcq+stride3q] + lea srcq, [srcq+strideq*4] + vpbroadcastq m4, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + vpbroadcastq m6, [srcq+strideq*2] + vpbroadcastq m7, [srcq+stride3q] + vpbroadcastd m8, [pw_128] + pxor m9, m9 + + vpblendd m0, m0, m7, 0xf0 + vpblendd m1, m1, m6, 0xf0 + vpblendd m2, m2, m5, 0xf0 + vpblendd m3, m3, m4, 0xf0 + + punpcklbw m0, m9 + punpcklbw m1, m9 + punpcklbw m2, m9 + punpcklbw m3, m9 + + psubw m0, m8 + psubw m1, m8 + psubw m2, m8 + psubw m3, m8 + + ; shuffle registers to generate partial_sum_diag[0-1] together + vpermq m7, m0, q1032 + vpermq m6, m1, q1032 + vpermq m5, m2, q1032 + vpermq m4, m3, q1032 + + ; start with partial_sum_hv[0-1] + paddw m8, m0, m1 + paddw m9, m2, m3 + phaddw m10, m0, m1 + phaddw m11, m2, m3 + paddw m8, m9 + phaddw m10, m11 + vextracti128 xm9, m8, 1 + vextracti128 xm11, m10, 1 + paddw xm8, xm9 ; partial_sum_hv[1] + phaddw xm10, xm11 ; partial_sum_hv[0] + vinserti128 m8, xm10, 1 + vpbroadcastd m9, [div_table+44] + pmaddwd m8, m8 + pmulld m8, m9 ; cost6[2a-d] | cost2[a-d] + + ; create aggregates [lower half]: + ; m9 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234+ + ; m4:xxxx0123+m5:xxxxx012+m6:xxxxxx01+m7:xxxxxxx0 + ; m10= m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx+ + ; m4:4567xxxx+m5:34567xxx+m6:234567xx+m7:1234567x + ; and [upper half]: + ; m9 = m0:xxxxxxx0+m1:xxxxxx01+m2:xxxxx012+m3:xxxx0123+ + ; m4:xxx01234+m5:xx012345+m6:x0123456+m7:01234567 + ; m10= m0:1234567x+m1:234567xx+m2:34567xxx+m3:4567xxxx+ + ; m4:567xxxxx+m5:67xxxxxx+m6:7xxxxxxx + ; and then shuffle m11 [shufw_6543210x], unpcklwd, pmaddwd, pmulld, paddd + + pslldq m9, m1, 2 + psrldq m10, m1, 14 + pslldq m11, m2, 4 + psrldq m12, m2, 12 + pslldq m13, m3, 6 + psrldq m14, m3, 10 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 + pslldq m11, m4, 8 + psrldq m12, m4, 8 + pslldq m13, m5, 10 + psrldq m14, m5, 6 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 + pslldq m11, m6, 12 + psrldq m12, m6, 4 + pslldq m13, m7, 14 + psrldq m14, m7, 2 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 ; partial_sum_diag[0/1][8-14,zero] + vbroadcasti128 m14, [shufw_6543210x] + vbroadcasti128 m13, [div_table+16] + vbroadcasti128 m12, [div_table+0] + paddw m9, m0 ; partial_sum_diag[0/1][0-7] + pshufb m10, m14 + punpckhwd m11, m9, m10 + punpcklwd m9, m10 + pmaddwd m11, m11 + pmaddwd m9, m9 + pmulld m11, m13 + pmulld m9, m12 + paddd m9, m11 ; cost0[a-d] | cost4[a-d] + + ; merge horizontally and vertically for partial_sum_alt[0-3] + paddw m10, m0, m1 + paddw m11, m2, m3 + paddw m12, m4, m5 + paddw m13, m6, m7 + phaddw m0, m4 + phaddw m1, m5 + phaddw m2, m6 + phaddw m3, m7 + + ; create aggregates [lower half]: + ; m4 = m10:01234567+m11:x0123456+m12:xx012345+m13:xxx01234 + ; m11= m11:7xxxxxxx+m12:67xxxxxx+m13:567xxxxx + ; and [upper half]: + ; m4 = m10:xxx01234+m11:xx012345+m12:x0123456+m13:01234567 + ; m11= m10:567xxxxx+m11:67xxxxxx+m12:7xxxxxxx + ; and then pshuflw m11 3012, unpcklwd, pmaddwd, pmulld, paddd + + pslldq m4, m11, 2 + psrldq m11, 14 + pslldq m5, m12, 4 + psrldq m12, 12 + pslldq m6, m13, 6 + psrldq m13, 10 + paddw m4, m10 + paddw m11, m12 + vpbroadcastd m12, [div_table+44] + paddw m5, m6 + paddw m11, m13 ; partial_sum_alt[3/2] right + vbroadcasti128 m13, [div_table+32] + paddw m4, m5 ; partial_sum_alt[3/2] left + pshuflw m5, m11, q3012 + punpckhwd m6, m11, m4 + punpcklwd m4, m5 + pmaddwd m6, m6 + pmaddwd m4, m4 + pmulld m6, m12 + pmulld m4, m13 + paddd m4, m6 ; cost7[a-d] | cost5[a-d] + + ; create aggregates [lower half]: + ; m5 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234 + ; m1 = m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx + ; and [upper half]: + ; m5 = m0:xxx01234+m1:xx012345+m2:x0123456+m3:01234567 + ; m1 = m0:567xxxxx+m1:67xxxxxx+m2:7xxxxxxx + ; and then pshuflw m1 3012, unpcklwd, pmaddwd, pmulld, paddd + + pslldq m5, m1, 2 + psrldq m1, 14 + pslldq m6, m2, 4 + psrldq m2, 12 + pslldq m7, m3, 6 + psrldq m3, 10 + paddw m5, m0 + paddw m1, m2 + paddw m6, m7 + paddw m1, m3 ; partial_sum_alt[0/1] right + paddw m5, m6 ; partial_sum_alt[0/1] left + pshuflw m0, m1, q3012 + punpckhwd m1, m5 + punpcklwd m5, m0 + pmaddwd m1, m1 + pmaddwd m5, m5 + pmulld m1, m12 + pmulld m5, m13 + paddd m5, m1 ; cost1[a-d] | cost3[a-d] + + mova xm0, [pd_47130256+ 16] + mova m1, [pd_47130256] + phaddd m9, m8 + phaddd m5, m4 + phaddd m9, m5 + vpermd m0, m9 ; cost[0-3] + vpermd m1, m9 ; cost[4-7] | cost[0-3] + + ; now find the best cost + pmaxsd xm2, xm0, xm1 + pshufd xm3, xm2, q1032 + pmaxsd xm2, xm3 + pshufd xm3, xm2, q2301 + pmaxsd xm2, xm3 ; best cost + + ; find the idx using minpos + ; make everything other than the best cost negative via subtraction + ; find the min of unsigned 16-bit ints to sort out the negative values + psubd xm4, xm1, xm2 + psubd xm3, xm0, xm2 + packssdw xm3, xm4 + phminposuw xm3, xm3 + + ; convert idx to 32-bits + psrld xm3, 16 + movd eax, xm3 + + ; get idx^4 complement + vpermd m3, m1 + psubd xm2, xm3 + psrld xm2, 10 + movd [varq], xm2 + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_avx512.asm b/third_party/dav1d/src/x86/cdef_avx512.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef_avx512.asm @@ -0,0 +1,867 @@ +; Copyright © 2020, VideoLAN and dav1d authors +; Copyright © 2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +%if HAVE_AVX512ICL && ARCH_X86_64 + +%macro DUP4 1-* + %rep %0 + times 4 db %1 + %rotate 1 + %endrep +%endmacro + +%macro DIRS 16 ; cdef_directions[] + %rep 4 + 16 + 4 ; 6 7 0 1 2 3 4 5 6 7 0 1 + ; masking away unused bits allows us to use a single vpaddd {1to16} + ; instruction instead of having to do vpbroadcastd + paddb + db %13 & 0x3f, -%13 & 0x3f + %rotate 1 + %endrep +%endmacro + +SECTION_RODATA 64 + +lut_perm_4x4: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + db 16, 17, 0, 1, 2, 3, 4, 5, 18, 19, 8, 9, 10, 11, 12, 13 + db 20, 21, 80, 81, 82, 83, 84, 85, 22, 23, 32, 33, 34, 35, 36, 37 + db 98, 99,100,101,102,103,104,105, 50, 51, 52, 53, 54, 55, 56, 57 +lut_perm_4x8a: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + db 96, 97, 0, 1, 2, 3, 4, 5, 98, 99, 8, 9, 10, 11, 12, 13 +lut_perm_4x8b:db 100,101, 16, 17, 18, 19, 20, 21,102,103, 24, 25, 26, 27, 28, 29 + db 104,105, 32, 33, 34, 35, 36, 37,106,107, 40, 41, 42, 43, 44, 45 + db 108,109, 48, 49, 50, 51, 52, 53,110,111, 56, 57, 58, 59, 60, 61 + db 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95 +pd_01234567: dd 0, 1, 2, 3, 4, 5, 6, 7 +lut_perm_8x8a: db 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 + db -1, -1, 34, 35, 36, 37, 38, 39, -1, -1, 50, 51, 52, 53, 54, 55 + db -1, -1, 66, 67, 68, 69, 70, 71, -1, -1, 82, 83, 84, 85, 86, 87 + db 96, 97, 98, 99,100,101,102,103,112,113,114,115,116,117,118,119 +lut_perm_8x8b: db 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23, 24, 25, 26, 27 + db 36, 37, 38, 39, 40, 41, 42, 43, 52, 53, 54, 55, 56, 57, 58, 59 + db 68, 69, 70, 71, 72, 73, 74, 75, 84, 85, 86, 87, 88, 89, 90, 91 + db 100,101,102,103,104,105,106,107,116,117,118,119,120,121,122,123 +edge_mask: dq 0x00003c3c3c3c0000, 0x00003f3f3f3f0000 ; 0000, 0001 + dq 0x0000fcfcfcfc0000, 0x0000ffffffff0000 ; 0010, 0011 + dq 0x00003c3c3c3c3c3c, 0x00003f3f3f3f3f3f ; 0100, 0101 + dq 0x0000fcfcfcfcfcfc, 0x0000ffffffffffff ; 0110, 0111 + dq 0x3c3c3c3c3c3c0000, 0x3f3f3f3f3f3f0000 ; 1000, 1001 + dq 0xfcfcfcfcfcfc0000, 0xffffffffffff0000 ; 1010, 1011 + dq 0x3c3c3c3c3c3c3c3c, 0x3f3f3f3f3f3f3f3f ; 1100, 1101 + dq 0xfcfcfcfcfcfcfcfc, 0xffffffffffffffff ; 1110, 1111 +px_idx: DUP4 18, 19, 20, 21, 26, 27, 28, 29, 34, 35, 36, 37, 42, 43, 44, 45 +cdef_dirs: DIRS -7,-14, 1, -6, 1, 2, 1, 10, 9, 18, 8, 17, 8, 16, 8, 15 +gf_shr: dq 0x0102040810204080, 0x0102040810204080 ; >> 0, >> 0 + dq 0x0204081020408000, 0x0408102040800000 ; >> 1, >> 2 + dq 0x0810204080000000, 0x1020408000000000 ; >> 3, >> 4 + dq 0x2040800000000000, 0x4080000000000000 ; >> 5, >> 6 + times 16 db 0 ; realign (introduced by cdef_dirs) +end_perm_w8clip:db 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30 + db 32, 36, 40, 44, 34, 38, 42, 46, 48, 52, 56, 60, 50, 54, 58, 62 + db 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31 + db 33, 37, 41, 45, 35, 39, 43, 47, 49, 53, 57, 61, 51, 55, 59, 63 +end_perm: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 +pri_tap: db 64, 64, 32, 32, 48, 48, 48, 48 ; left-shifted by 4 +sec_tap: db 32, 32, 16, 16 +pd_268435568: dd 268435568 + +SECTION .text + +%if WIN64 +DECLARE_REG_TMP 5, 6 +%else +DECLARE_REG_TMP 8, 5 +%endif + +; lut: +; t0 t1 t2 t3 t4 t5 t6 t7 +; T0 T1 T2 T3 T4 T5 T6 T7 +; L0 L1 00 01 02 03 04 05 +; L2 L3 10 11 12 13 14 15 +; L4 L5 20 21 22 23 24 25 +; L6 L7 30 31 32 33 34 35 +; 4e 4f 40 41 42 43 44 45 +; 5e 5f 50 51 52 53 54 55 + +INIT_ZMM avx512icl +cglobal cdef_filter_4x4, 4, 8, 13, dst, stride, left, top, pri, sec, dir, damping, edge +%define base r7-edge_mask + movq xmm0, [dstq+strideq*0] + movhps xmm0, [dstq+strideq*1] + lea r7, [edge_mask] + movq xmm1, [topq+strideq*0-2] + movhps xmm1, [topq+strideq*1-2] + mov r6d, edgem + vinserti32x4 ym0, ymm0, [leftq], 1 + lea r2, [strideq*3] + vinserti32x4 ym1, ymm1, [dstq+strideq*2], 1 + mova m5, [base+lut_perm_4x4] + vinserti32x4 m0, [dstq+r2], 2 + test r6b, 0x08 ; avoid buffer overread + jz .main + lea r3, [dstq+strideq*4-4] + vinserti32x4 m1, [r3+strideq*0], 2 + vinserti32x4 m0, [r3+strideq*1], 3 +.main: + movifnidn prid, prim + mov t0d, dirm + mova m3, [base+px_idx] + mov r3d, dampingm + vpermi2b m5, m0, m1 ; lut + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m7, m7 + lea r3, [r7+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m6, m3, m5 ; px + cmp r6d, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m1, m3, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m1, m1, m5 ; k0p0 k0p1 k1p0 k1p1 +%macro CDEF_FILTER_4x4_PRI 0 + vpcmpub k1, m6, m1, 6 ; px > pN + psubb m2, m1, m6 + lzcnt r6d, prid + vpsubb m2{k1}, m6, m1 ; abs(diff) + vpbroadcastb m4, prid + and prid, 1 + vgf2p8affineqb m9, m2, [r3+r6*8] {1to8}, 0 ; abs(diff) >> shift + movifnidn t1d, secm + vpbroadcastd m10, [base+pri_tap+priq*4] + vpsubb m10{k1}, m7, m10 ; apply_sign(pri_tap) + psubusb m4, m9 ; imax(0, pri_strength - (abs(diff) >> shift))) + pminub m2, m4 + vpdpbusd m0, m2, m10 ; sum +%endmacro + CDEF_FILTER_4x4_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m4, m6, m1 + pmaxub m1, m6 + pminub m5, m2, m3 + pmaxub m2, m3 + pminub m4, m5 + pmaxub m2, m1 + psrldq m1, m4, 2 + psrldq m3, m2, 2 + pminub m1, m4 + vpcmpw k1, m0, m7, 1 + vpshldd m6, m0, 8 + pmaxub m2, m3 + pslldq m3, m1, 1 + psubw m7, m0 + paddusw m0, m6 ; clip >0xff + vpsubusw m0{k1}, m6, m7 ; clip <0x00 + pslldq m4, m2, 1 + pminub m1, m3 + pmaxub m2, m4 + pmaxub m0, m1 + pminub m0, m2 + jmp .end +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + vpshldd m6, m0, 8 ; (px << 8) + ((sum > -8) << 4) + paddw m0, m6 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) +.end: + mova xm1, [base+end_perm] + vpermb m0, m1, m0 ; output in bits 8-15 of each dword + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r2 ], xm0, 3 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + vpbroadcastq m8, [base+edge_mask+r6*8] + test prid, prid + jz .mask_edges_sec_only + vpaddd m2, m3, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m8, m2 ; index in-range + mova m1, m6 + vpermb m1{k1}, m2, m5 + CDEF_FILTER_4x4_PRI + test t1d, t1d + jz .end_no_clip + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m4, m3, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m9, m3, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m8, m4 + mova m2, m6 + vpermb m2{k1}, m4, m5 + vpshufbitqmb k1, m8, m9 + mova m3, m6 + vpermb m3{k1}, m9, m5 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m2, m3, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m3, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m2, m2, m5 ; k0s0 k0s1 k1s0 k1s1 + vpermb m3, m3, m5 ; k0s2 k0s3 k1s2 k1s3 +.sec_main: + vpbroadcastd m8, [base+sec_tap] + vpcmpub k1, m6, m2, 6 + psubb m4, m2, m6 + vpbroadcastb m12, t1d + lzcnt t1d, t1d + vpsubb m4{k1}, m6, m2 + vpcmpub k2, m6, m3, 6 + vpbroadcastq m11, [r3+t1*8] + gf2p8affineqb m10, m4, m11, 0 + psubb m5, m3, m6 + mova m9, m8 + vpsubb m8{k1}, m7, m8 + psubusb m10, m12, m10 + vpsubb m5{k2}, m6, m3 + pminub m4, m10 + vpdpbusd m0, m4, m8 + gf2p8affineqb m11, m5, m11, 0 + vpsubb m9{k2}, m7, m9 + psubusb m12, m11 + pminub m5, m12 + vpdpbusd m0, m5, m9 + ret + +DECLARE_REG_TMP 2, 7 + +; lut top lut bottom +; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25 +; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35 +; L0 L1 00 01 02 03 04 05 L8 L9 40 41 42 43 44 45 +; L2 L3 10 11 12 13 14 15 La Lb 50 51 52 53 54 55 +; L4 L5 20 21 22 23 24 25 Lc Ld 60 61 62 63 64 65 +; L6 L7 30 31 32 33 34 35 Le Lf 70 71 72 73 74 75 +; L8 L9 40 41 42 43 44 45 8e 8f 80 81 82 83 84 85 +; La Lb 50 51 52 53 54 55 9e 9f 90 91 92 93 94 95 + +cglobal cdef_filter_4x8, 4, 9, 22, dst, stride, left, top, \ + pri, sec, dir, damping, edge +%define base r8-edge_mask + vpbroadcastd ym21, strided + mov r6d, edgem + lea r8, [edge_mask] + movq xm1, [topq+strideq*0-2] + pmulld ym21, [base+pd_01234567] + kxnorb k1, k1, k1 + movq xm2, [topq+strideq*1-2] + vpgatherdq m0{k1}, [dstq+ym21] ; +0+1 +2+3 +4+5 +6+7 + mova m14, [base+lut_perm_4x8a] + movu m15, [base+lut_perm_4x8b] + test r6b, 0x08 ; avoid buffer overread + jz .main + lea r7, [dstq+strideq*8-2] + vinserti32x4 ym1, [r7+strideq*0], 1 + vinserti32x4 ym2, [r7+strideq*1], 1 +.main: + punpcklqdq ym1, ym2 + vinserti32x4 m1, [leftq], 2 ; -2-1 +8+9 left ____ + movifnidn prid, prim + mov t0d, dirm + mova m16, [base+px_idx] + mov r3d, dampingm + vpermi2b m14, m0, m1 ; lut top + vpermi2b m15, m0, m1 ; lut bottom + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m20, m20 + lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m2, m16, m14 ; pxt + vpermb m3, m16, m15 ; pxb + mova m1, m0 + cmp r6b, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m4, m6, m14 ; pNt k0p0 k0p1 k1p0 k1p1 + vpermb m5, m6, m15 ; pNb +%macro CDEF_FILTER_4x8_PRI 0 + vpcmpub k1, m2, m4, 6 ; pxt > pNt + vpcmpub k2, m3, m5, 6 ; pxb > pNb + psubb m6, m4, m2 + psubb m7, m5, m3 + lzcnt r6d, prid + vpsubb m6{k1}, m2, m4 ; abs(diff_top) + vpsubb m7{k2}, m3, m5 ; abs(diff_bottom) + vpbroadcastb m13, prid + vpbroadcastq m9, [r3+r6*8] + and prid, 1 + vpbroadcastd m11, [base+pri_tap+priq*4] + vgf2p8affineqb m8, m6, m9, 0 ; abs(dt) >> shift + vgf2p8affineqb m9, m7, m9, 0 ; abs(db) >> shift + mova m10, m11 + movifnidn t1d, secm + vpsubb m10{k1}, m20, m11 ; apply_sign(pri_tap_top) + vpsubb m11{k2}, m20, m11 ; apply_sign(pri_tap_bottom) + psubusb m12, m13, m8 ; imax(0, pri_strength - (abs(dt) >> shift))) + psubusb m13, m13, m9 ; imax(0, pri_strength - (abs(db) >> shift))) + pminub m6, m12 + pminub m7, m13 + vpdpbusd m0, m6, m10 ; sum top + vpdpbusd m1, m7, m11 ; sum bottom +%endmacro + CDEF_FILTER_4x8_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m10, m4, m2 + pminub m12, m6, m8 + pminub m11, m5, m3 + pminub m13, m7, m9 + pmaxub m4, m2 + pmaxub m6, m8 + pmaxub m5, m3 + pmaxub m7, m9 + pminub m10, m12 + pminub m11, m13 + pmaxub m4, m6 + pmaxub m5, m7 + mov r2d, 0xAAAAAAAA + kmovd k1, r2d + kxnorb k2, k2, k2 ; hw lw + vpshrdd m12, m0, m1, 16 ; m1lw m0hw + vpshrdd m6, m10, m11, 16 ; m11lw m10hw + vpshrdd m8, m4, m5, 16 ; m5lw m4hw + vpblendmw m7{k1}, m10, m11 ; m11hw m10lw + vpblendmw m9{k1}, m4, m5 ; m5hw m4lw + vpblendmw m4{k1}, m0, m12 ; m1lw m0lw + vpblendmw m5{k1}, m12, m1 ; m1hw m0hw + vpshrdd m2, m3, 16 + pminub m6, m7 + pmaxub m8, m9 + mova ym14, [base+end_perm] + vpcmpw k1, m4, m20, 1 + vpshldw m2, m5, 8 + pslldq m7, m6, 1 + pslldq m9, m8, 1 + psubw m5, m20, m4 + paddusw m0, m4, m2 ; clip >0xff + pminub m6, m7 + pmaxub m8, m9 + psubusw m0{k1}, m2, m5 ; clip <0x00 + pmaxub m0, m6 + pminub m0, m8 + vpermb m0, m14, m0 + vpscatterdd [dstq+ym21]{k2}, ym0 + RET +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + mova ym4, [base+end_perm] + kxnorb k1, k1, k1 + vpshldd m2, m0, 8 ; (px << 8) + ((sum > -8) << 4) + vpshldd m3, m1, 8 + paddw m0, m2 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + paddw m1, m3 + pslld m0, 16 + vpshrdd m0, m1, 16 + vpermb m0, m4, m0 ; output in bits 8-15 of each word + vpscatterdd [dstq+ym21]{k1}, ym0 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + mov t1d, r6d + or r6d, 8 ; top 4x4 has bottom + or t1d, 4 ; bottom 4x4 has top + vpbroadcastq m17, [base+edge_mask+r6*8] + vpbroadcastq m18, [base+edge_mask+t1*8] + test prid, prid + jz .mask_edges_sec_only + vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m17, m6 ; index in-range + vpshufbitqmb k2, m18, m6 + mova m4, m2 + mova m5, m3 + vpermb m4{k1}, m6, m14 + vpermb m5{k2}, m6, m15 + CDEF_FILTER_4x8_PRI + test t1d, t1d + jz .end_no_clip + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m10, m16, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m11, m16, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m17, m10 + vpshufbitqmb k2, m18, m10 + vpshufbitqmb k3, m17, m11 + vpshufbitqmb k4, m18, m11 + mova m6, m2 + mova m7, m3 + mova m8, m2 + mova m9, m3 + vpermb m6{k1}, m10, m14 + vpermb m7{k2}, m10, m15 + vpermb m8{k3}, m11, m14 + vpermb m9{k4}, m11, m15 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m8, m16, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m9, m16, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m6, m8, m14 ; pNt k0s0 k0s1 k1s0 k1s1 + vpermb m7, m8, m15 ; pNb + vpermb m8, m9, m14 ; pNt k0s2 k0s3 k1s2 k1s3 + vpermb m9, m9, m15 ; pNb +.sec_main: + vpbroadcastb m18, t1d + lzcnt t1d, t1d + vpcmpub k1, m2, m6, 6 + vpcmpub k2, m3, m7, 6 + vpcmpub k3, m2, m8, 6 + vpcmpub k4, m3, m9, 6 + vpbroadcastq m17, [r3+t1*8] + psubb m10, m6, m2 + psubb m11, m7, m3 + psubb m12, m8, m2 + psubb m13, m9, m3 + vpsubb m10{k1}, m2, m6 ; abs(dt0) + vpsubb m11{k2}, m3, m7 ; abs(db0) + vpsubb m12{k3}, m2, m8 ; abs(dt1) + vpsubb m13{k4}, m3, m9 ; abs(db1) + vpbroadcastd m19, [base+sec_tap] + gf2p8affineqb m14, m10, m17, 0 ; abs(dt0) >> shift + gf2p8affineqb m15, m11, m17, 0 ; abs(db0) >> shift + gf2p8affineqb m16, m12, m17, 0 ; abs(dt1) >> shift + gf2p8affineqb m17, m13, m17, 0 ; abs(db1) >> shift + psubusb m14, m18, m14 ; imax(0, sec_strength - (abs(dt0) >> shift))) + psubusb m15, m18, m15 ; imax(0, sec_strength - (abs(db0) >> shift))) + psubusb m16, m18, m16 ; imax(0, sec_strength - (abs(dt1) >> shift))) + psubusb m17, m18, m17 ; imax(0, sec_strength - (abs(db1) >> shift))) + pminub m10, m14 + pminub m11, m15 + pminub m12, m16 + pminub m13, m17 + mova m14, m19 + mova m15, m19 + mova m16, m19 + vpsubb m14{k1}, m20, m19 ; apply_sign(sec_tap_top_0) + vpsubb m15{k2}, m20, m19 ; apply_sign(sec_tap_bottom_0) + vpsubb m16{k3}, m20, m19 ; apply_sign(sec_tap_top_1) + vpsubb m19{k4}, m20, m19 ; apply_sign(sec_tap_bottom_1) + vpdpbusd m0, m10, m14 + vpdpbusd m1, m11, m15 + vpdpbusd m0, m12, m16 + vpdpbusd m1, m13, m19 + ret + +; lut tl lut tr +; t0 t1 t2 t3 t4 t5 t6 t7 t6 t7 t8 t9 ta tb tc td +; T0 T1 T2 T3 T4 T5 T6 T7 T6 T7 T8 T9 TA TB TC TD +; L0 L1 00 01 02 03 04 05 04 05 06 07 08 09 0a 0b +; L2 L3 10 11 12 13 14 15 14 15 16 17 18 19 1a 1b +; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b +; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b +; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b +; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b +; lut bl lut br +; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b +; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b +; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b +; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b +; Lc Ld 60 61 62 63 64 65 64 65 66 67 68 69 6a 6b +; Le Lf 70 71 72 73 74 75 74 75 76 77 78 79 7a 7b +; 8e 8f 80 81 82 83 84 85 84 85 86 87 88 89 8a 8b +; 9e 9f 90 91 92 93 94 95 94 95 96 97 98 99 9a 9b + +cglobal cdef_filter_8x8, 4, 11, 32, 4*64, dst, stride, left, top, \ + pri, sec, dir, damping, edge +%define base r8-edge_mask + mov r6d, edgem + lea r10, [dstq+strideq*4-2] + movu xmm0, [topq+strideq*0-2] + movu xmm1, [dstq+strideq*2-2] + movu xmm2, [r10 +strideq*2 ] + lea r8, [edge_mask] + lea r9, [strideq*3] + pmovzxwq m10, [leftq-4] + vinserti32x4 ym0, ymm0, [topq+strideq*1-2], 1 + vinserti32x4 ym1, ymm1, [dstq+r9 -2], 1 + vinserti32x4 ym2, ymm2, [r10 +r9 ], 1 + lea r7, [r10 +strideq*4 ] + pmovzxwq m11, [leftq+4] + vinserti32x4 m0, [dstq+strideq*0-2], 2 + vinserti32x4 m1, [r10 +strideq*0 ], 2 + mova m12, [base+lut_perm_8x8a] + movu m13, [base+lut_perm_8x8b] + vinserti32x4 m0, [dstq+strideq*1-2], 3 + vinserti32x4 m1, [r10 +strideq*1 ], 3 + test r6b, 0x08 ; avoid buffer overread + jz .main + vinserti32x4 m2, [r7 +strideq*0], 2 + vinserti32x4 m2, [r7 +strideq*1], 3 +.main: + mov t1d, 0x11111100 + mova m14, m12 + mova m15, m13 + kmovd k1, t1d + kshiftrd k2, k1, 8 + movifnidn prid, prim + mov t0d, dirm + mova m30, [base+px_idx] + mov r3d, dampingm + vpermi2b m12, m0, m1 ; lut tl + vpermi2b m14, m1, m2 ; lut bl + vpermi2b m13, m0, m1 ; lut tr + vpermi2b m15, m1, m2 ; lut br + vpblendmw m12{k1}, m12, m10 + vpblendmw m14{k2}, m14, m11 + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m31, m31 + lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m4, m30, m12 ; pxtl + vpermb m5, m30, m13 ; pxtr + vpermb m6, m30, m14 ; pxbl + vpermb m7, m30, m15 ; pxbr + mova m1, m0 + mova m2, m0 + mova m3, m0 + cmp r6b, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m11, m30, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m8, m11, m12 ; pNtl k0p0 k0p1 k1p0 k1p1 + vpermb m9, m11, m13 ; pNtr + vpermb m10, m11, m14 ; pNbl + vpermb m11, m11, m15 ; pNbr +%macro CDEF_FILTER_8x8_PRI 0 + vpcmpub k1, m4, m8, 6 ; pxtl > pNtl + vpcmpub k2, m5, m9, 6 ; pxtr > pNtr + vpcmpub k3, m6, m10, 6 ; pxbl > pNbl + vpcmpub k4, m7, m11, 6 ; pxbr > pNbr + psubb m16, m8, m4 + psubb m17, m9, m5 + psubb m18, m10, m6 + psubb m19, m11, m7 + lzcnt r6d, prid + vpsubb m16{k1}, m4, m8 ; abs(diff_tl) + vpsubb m17{k2}, m5, m9 ; abs(diff_tr) + vpsubb m18{k3}, m6, m10 ; abs(diff_bl) + vpsubb m19{k4}, m7, m11 ; abs(diff_br) + vpbroadcastq m28, [r3+r6*8] + vpbroadcastb m29, prid + and prid, 1 + vpbroadcastd m27, [base+pri_tap+priq*4] + vgf2p8affineqb m20, m16, m28, 0 ; abs(dtl) >> shift + vgf2p8affineqb m21, m17, m28, 0 ; abs(dtr) >> shift + vgf2p8affineqb m22, m18, m28, 0 ; abs(dbl) >> shift + vgf2p8affineqb m23, m19, m28, 0 ; abs(dbl) >> shift + mova m24, m27 + mova m25, m27 + mova m26, m27 + movifnidn t1d, secm + vpsubb m24{k1}, m31, m27 ; apply_sign(pri_tap_tl) + vpsubb m25{k2}, m31, m27 ; apply_sign(pri_tap_tr) + vpsubb m26{k3}, m31, m27 ; apply_sign(pri_tap_tl) + vpsubb m27{k4}, m31, m27 ; apply_sign(pri_tap_tr) + psubusb m20, m29, m20 ; imax(0, pri_strength - (abs(dtl) >> shift))) + psubusb m21, m29, m21 ; imax(0, pri_strength - (abs(dtr) >> shift))) + psubusb m22, m29, m22 ; imax(0, pri_strength - (abs(dbl) >> shift))) + psubusb m23, m29, m23 ; imax(0, pri_strength - (abs(dbr) >> shift))) + pminub m16, m20 + pminub m17, m21 + pminub m18, m22 + pminub m19, m23 + vpdpbusd m0, m16, m24 ; sum tl + vpdpbusd m1, m17, m25 ; sum tr + vpdpbusd m2, m18, m26 ; sum bl + vpdpbusd m3, m19, m27 ; sum br +%endmacro + CDEF_FILTER_8x8_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m20, m8, m4 + pminub m24, m12, m16 + pminub m21, m9, m5 + pminub m25, m13, m17 + pminub m22, m10, m6 + pminub m26, m14, m18 + pminub m23, m11, m7 + pminub m27, m15, m19 + pmaxub m8, m4 + pmaxub m12, m16 + pmaxub m9, m5 + pmaxub m13, m17 + pmaxub m10, m6 + pmaxub m14, m18 + pmaxub m11, m7 + pmaxub m15, m19 + pminub m20, m24 + pminub m21, m25 + pminub m22, m26 + pminub m23, m27 + pmaxub m8, m12 + pmaxub m9, m13 + pmaxub m10, m14 + pmaxub m11, m15 + mov r2d, 0xAAAAAAAA + kmovd k1, r2d + vpshrdd m24, m0, m1, 16 + vpshrdd m25, m2, m3, 16 + vpshrdd m12, m20, m21, 16 + vpshrdd m14, m22, m23, 16 + vpshrdd m16, m8, m9, 16 + vpshrdd m18, m10, m11, 16 + vpblendmw m13{k1}, m20, m21 + vpblendmw m15{k1}, m22, m23 + vpblendmw m17{k1}, m8, m9 + vpblendmw m19{k1}, m10, m11 + vpblendmw m20{k1}, m0, m24 + vpblendmw m21{k1}, m24, m1 + vpblendmw m22{k1}, m2, m25 + vpblendmw m23{k1}, m25, m3 + vpshrdd m4, m5, 16 + vpshrdd m6, m7, 16 + pminub m12, m13 + pminub m14, m15 + pmaxub m16, m17 + pmaxub m18, m19 + mova m8, [base+end_perm_w8clip] + vpcmpw k2, m20, m31, 1 + vpcmpw k3, m22, m31, 1 + vpshldw m4, m21, 8 + vpshldw m6, m23, 8 + kunpckdq k1, k1, k1 + kxnorb k4, k4, k4 + vpshrdw m11, m12, m14, 8 + vpshrdw m15, m16, m18, 8 + vpblendmb m13{k1}, m12, m14 + vpblendmb m17{k1}, m16, m18 + psubw m21, m31, m20 + psubw m23, m31, m22 + paddusw m0, m20, m4 ; clip >0xff + paddusw m1, m22, m6 + pminub m11, m13 + pmaxub m15, m17 + psubusw m0{k2}, m4, m21 ; clip <0x00 + psubusw m1{k3}, m6, m23 + psrlw m0, 8 + vmovdqu8 m0{k1}, m1 + pmaxub m0, m11 + pminub m0, m15 + vpermb m0, m8, m0 + add r10, 2 + vextracti32x4 xm1, m0, 1 + vextracti32x4 xm2, m0, 2 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*2], xm1 + movq [r10 +strideq*0], xm2 + movq [r10 +strideq*2], xm3 + movhps [dstq+strideq*1], xm0 + movhps [dstq+r9 ], xm1 + movhps [r10 +strideq*1], xm2 + movhps [r10 +r9 ], xm3 + RET +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + mova xm8, [base+end_perm] + kxnorb k1, k1, k1 + vpshldd m4, m0, 8 ; (px << 8) + ((sum > -8) << 4) + vpshldd m5, m1, 8 + vpshldd m6, m2, 8 + vpshldd m7, m3, 8 + paddw m0, m4 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + paddw m1, m5 + paddw m2, m6 + paddw m3, m7 + vpermb m0, m8, m0 + vpermb m1, m8, m1 + vpermb m2, m8, m2 + vpermb m3, m8, m3 + add r10, 2 + punpckldq m4, m0, m1 + punpckhdq m0, m1 + punpckldq m5, m2, m3 + punpckhdq m2, m3 + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*2], xm0 + movq [r10 +strideq*0], xm5 + movq [r10 +strideq*2], xm2 + movhps [dstq+strideq*1], xm4 + movhps [dstq+r9 ], xm0 + movhps [r10 +strideq*1], xm5 + movhps [r10 +r9 ], xm2 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + mov t0d, r6d + mov t1d, r6d + or t0d, 0xA ; top-left 4x4 has bottom and right + or t1d, 0x9 ; top-right 4x4 has bottom and left + vpbroadcastq m26, [base+edge_mask+t0*8] + vpbroadcastq m27, [base+edge_mask+t1*8] + mov t1d, r6d + or r6d, 0x6 ; bottom-left 4x4 has top and right + or t1d, 0x5 ; bottom-right 4x4 has top and left + vpbroadcastq m28, [base+edge_mask+r6*8] + vpbroadcastq m29, [base+edge_mask+t1*8] + mov t0d, dirm + test prid, prid + jz .mask_edges_sec_only + vpaddd m20, m30, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m26, m20 ; index in-range + vpshufbitqmb k2, m27, m20 + vpshufbitqmb k3, m28, m20 + vpshufbitqmb k4, m29, m20 + mova m8, m4 + mova m9, m5 + mova m10, m6 + mova m11, m7 + vpermb m8{k1}, m20, m12 + vpermb m9{k2}, m20, m13 + vpermb m10{k3}, m20, m14 + vpermb m11{k4}, m20, m15 + mova [rsp+0x00], m26 + mova [rsp+0x40], m27 + mova [rsp+0x80], m28 + mova [rsp+0xC0], m29 + CDEF_FILTER_8x8_PRI + test t1d, t1d + jz .end_no_clip + mova m26, [rsp+0x00] + mova m27, [rsp+0x40] + mova m28, [rsp+0x80] + mova m29, [rsp+0xC0] + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m26, m20 + vpshufbitqmb k2, m27, m20 + vpshufbitqmb k3, m28, m20 + vpshufbitqmb k4, m29, m20 + mova m16, m4 + mova m17, m5 + mova m18, m6 + mova m19, m7 + vpermb m16{k1}, m20, m12 + vpermb m17{k2}, m20, m13 + vpermb m18{k3}, m20, m14 + vpermb m19{k4}, m20, m15 + vpshufbitqmb k1, m26, m21 + vpshufbitqmb k2, m27, m21 + vpshufbitqmb k3, m28, m21 + vpshufbitqmb k4, m29, m21 + vpermb m12, m21, m12 + vpermb m13, m21, m13 + vpermb m14, m21, m14 + vpermb m15, m21, m15 + vpblendmb m12{k1}, m4, m12 + vpblendmb m13{k2}, m5, m13 + vpblendmb m14{k3}, m6, m14 + vpblendmb m15{k4}, m7, m15 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m16, m20, m12 ; pNtl k0s0 k0s1 k1s0 k1s1 + vpermb m17, m20, m13 ; pNtr + vpermb m18, m20, m14 ; pNbl + vpermb m19, m20, m15 ; pNbr + vpermb m12, m21, m12 ; pNtl k0s2 k0s3 k1s2 k1s3 + vpermb m13, m21, m13 ; pNtr + vpermb m14, m21, m14 ; pNbl + vpermb m15, m21, m15 ; pNbr +.sec_main: +%macro CDEF_FILTER_8x8_SEC 4-5 0 ; load constants + vpcmpub k1, m4, %1, 6 + vpcmpub k2, m5, %2, 6 + vpcmpub k3, m6, %3, 6 + vpcmpub k4, m7, %4, 6 + psubb m20, %1, m4 + psubb m21, %2, m5 + psubb m22, %3, m6 + psubb m23, %4, m7 +%if %5 + vpbroadcastb m28, t1d + lzcnt t1d, t1d + vpbroadcastq m29, [r3+t1*8] +%endif + vpsubb m20{k1}, m4, %1 + vpsubb m21{k2}, m5, %2 + vpsubb m22{k3}, m6, %3 + vpsubb m23{k4}, m7, %4 + gf2p8affineqb m24, m20, m29, 0 + gf2p8affineqb m25, m21, m29, 0 + gf2p8affineqb m26, m22, m29, 0 + gf2p8affineqb m27, m23, m29, 0 +%if %5 + vpbroadcastd m30, [base+sec_tap] +%endif + psubusb m24, m28, m24 + psubusb m25, m28, m25 + psubusb m26, m28, m26 + psubusb m27, m28, m27 + pminub m20, m24 + pminub m21, m25 + pminub m22, m26 + pminub m23, m27 + mova m24, m30 + mova m25, m30 + mova m26, m30 + mova m27, m30 + vpsubb m24{k1}, m31, m30 + vpsubb m25{k2}, m31, m30 + vpsubb m26{k3}, m31, m30 + vpsubb m27{k4}, m31, m30 + vpdpbusd m0, m20, m24 + vpdpbusd m1, m21, m25 + vpdpbusd m2, m22, m26 + vpdpbusd m3, m23, m27 +%endmacro + CDEF_FILTER_8x8_SEC m16, m17, m18, m19, 1 + CDEF_FILTER_8x8_SEC m12, m13, m14, m15 + ret + +%endif ; HAVE_AVX512ICL && ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_init_tmpl.c b/third_party/dav1d/src/x86/cdef_init_tmpl.c --- a/third_party/dav1d/src/x86/cdef_init_tmpl.c +++ b/third_party/dav1d/src/x86/cdef_init_tmpl.c @@ -26,24 +26,20 @@ */ #include "src/cpu.h" #include "src/cdef.h" -decl_cdef_fn(dav1d_cdef_filter_8x8_avx2); -decl_cdef_fn(dav1d_cdef_filter_8x8_sse4); -decl_cdef_fn(dav1d_cdef_filter_8x8_ssse3); -decl_cdef_fn(dav1d_cdef_filter_8x8_sse2); +#define decl_cdef_size_fn(sz) \ + decl_cdef_fn(dav1d_cdef_filter_##sz##_avx512icl); \ + decl_cdef_fn(dav1d_cdef_filter_##sz##_avx2); \ + decl_cdef_fn(dav1d_cdef_filter_##sz##_sse4); \ + decl_cdef_fn(dav1d_cdef_filter_##sz##_ssse3); \ + decl_cdef_fn(dav1d_cdef_filter_##sz##_sse2) -decl_cdef_fn(dav1d_cdef_filter_4x8_avx2); -decl_cdef_fn(dav1d_cdef_filter_4x8_sse4); -decl_cdef_fn(dav1d_cdef_filter_4x8_ssse3); -decl_cdef_fn(dav1d_cdef_filter_4x8_sse2); - -decl_cdef_fn(dav1d_cdef_filter_4x4_avx2); -decl_cdef_fn(dav1d_cdef_filter_4x4_sse4); -decl_cdef_fn(dav1d_cdef_filter_4x4_ssse3); -decl_cdef_fn(dav1d_cdef_filter_4x4_sse2); +decl_cdef_size_fn(4x4); +decl_cdef_size_fn(4x8); +decl_cdef_size_fn(8x8); decl_cdef_dir_fn(dav1d_cdef_dir_avx2); decl_cdef_dir_fn(dav1d_cdef_dir_sse4); decl_cdef_dir_fn(dav1d_cdef_dir_ssse3); @@ -74,14 +70,25 @@ c->fb[0] = dav1d_cdef_filter_8x8_sse4; c->fb[1] = dav1d_cdef_filter_4x8_sse4; c->fb[2] = dav1d_cdef_filter_4x4_sse4; #endif +#if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; -#if BITDEPTH == 8 && ARCH_X86_64 +#if BITDEPTH == 8 c->dir = dav1d_cdef_dir_avx2; c->fb[0] = dav1d_cdef_filter_8x8_avx2; c->fb[1] = dav1d_cdef_filter_4x8_avx2; c->fb[2] = dav1d_cdef_filter_4x4_avx2; #endif + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + +#if HAVE_AVX512ICL && BITDEPTH == 8 + c->fb[0] = dav1d_cdef_filter_8x8_avx512icl; + c->fb[1] = dav1d_cdef_filter_4x8_avx512icl; + c->fb[2] = dav1d_cdef_filter_4x4_avx512icl; +#endif + +#endif } diff --git a/third_party/dav1d/src/x86/cdef_sse.asm b/third_party/dav1d/src/x86/cdef_sse.asm --- a/third_party/dav1d/src/x86/cdef_sse.asm +++ b/third_party/dav1d/src/x86/cdef_sse.asm @@ -362,30 +362,23 @@ mov dword [px+2*%3+%1*2], OUT_OF_BOUNDS mov dword [px+3*%3+%1*2], OUT_OF_BOUNDS .body_done: ; top - %if ARCH_X86_64 - DEFINE_ARGS dst, stride, left, top2, pri, sec, stride3, top1, edge - %else - DEFINE_ARGS dst, stride, left, top2, stride3, top1, edge - %endif LOAD_ARG32 top test edged, 4 ; have_top jz .no_top - mov top1q, [top2q+0*gprsize] - mov top2q, [top2q+1*gprsize] test edged, 1 ; have_left jz .top_no_left test edged, 2 ; have_right jz .top_no_right %if %1 == 4 - PMOVZXBW m0, [top1q-2] - PMOVZXBW m1, [top2q-2] + PMOVZXBW m0, [topq+strideq*0-2] + PMOVZXBW m1, [topq+strideq*1-2] %else - movu m0, [top1q-4] - movu m1, [top2q-4] + movu m0, [topq+strideq*0-4] + movu m1, [topq+strideq*1-4] punpckhbw m2, m0, m15 punpcklbw m0, m15 punpckhbw m3, m1, m15 punpcklbw m1, m15 movu [px-2*%3+8], m2 @@ -394,17 +387,17 @@ movu [px-2*%3-%1], m0 movu [px-1*%3-%1], m1 jmp .top_done .top_no_right: %if %1 == 4 - PMOVZXBW m0, [top1q-%1] - PMOVZXBW m1, [top2q-%1] + PMOVZXBW m0, [topq+strideq*0-%1] + PMOVZXBW m1, [topq+strideq*1-%1] movu [px-2*%3-4*2], m0 movu [px-1*%3-4*2], m1 %else - movu m0, [top1q-%1] - movu m1, [top2q-%2] + movu m0, [topq+strideq*0-%1] + movu m1, [topq+strideq*1-%2] punpckhbw m2, m0, m15 punpcklbw m0, m15 punpckhbw m3, m1, m15 punpcklbw m1, m15 mova [px-2*%3-8*2], m0 @@ -417,15 +410,15 @@ jmp .top_done .top_no_left: test edged, 2 ; have_right jz .top_no_left_right %if %1 == 4 - PMOVZXBW m0, [top1q] - PMOVZXBW m1, [top2q] + PMOVZXBW m0, [topq+strideq*0] + PMOVZXBW m1, [topq+strideq*1] %else - movu m0, [top1q] - movu m1, [top2q] + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] punpckhbw m2, m0, m15 punpcklbw m0, m15 punpckhbw m3, m1, m15 punpcklbw m1, m15 movd [px-2*%3+8*2], m2 @@ -435,12 +428,12 @@ mova [px-1*%3], m1 mov dword [px-2*%3-4], OUT_OF_BOUNDS mov dword [px-1*%3-4], OUT_OF_BOUNDS jmp .top_done .top_no_left_right: - PMOVZXBW m0, [top1q], %1 == 4 - PMOVZXBW m1, [top2q], %1 == 4 + PMOVZXBW m0, [topq+strideq*0], %1 == 4 + PMOVZXBW m1, [topq+strideq*1], %1 == 4 mova [px-2*%3], m0 mova [px-1*%3], m1 mov dword [px-2*%3+%1*2], OUT_OF_BOUNDS mov dword [px-1*%3+%1*2], OUT_OF_BOUNDS mov dword [px-2*%3-4], OUT_OF_BOUNDS @@ -628,13 +621,13 @@ bsr secdmpd, secdmpd sub pridmpd, dampingd sub secdmpd, dampingd xor dampingd, dampingd neg pridmpd - cmovl pridmpd, dampingd + cmovs pridmpd, dampingd neg secdmpd - cmovl secdmpd, dampingd + cmovs secdmpd, dampingd %if ARCH_X86_64 mov [rsp+ 0], pridmpq ; pri_shift mov [rsp+16], secdmpq ; sec_shift %else mov [esp+0x00], pridmpd diff --git a/third_party/dav1d/src/x86/cpu.h b/third_party/dav1d/src/x86/cpu.h --- a/third_party/dav1d/src/x86/cpu.h +++ b/third_party/dav1d/src/x86/cpu.h @@ -27,19 +27,16 @@ #ifndef DAV1D_SRC_X86_CPU_H #define DAV1D_SRC_X86_CPU_H enum CpuFlags { - DAV1D_X86_CPU_FLAG_SSE = 1 << 0, - DAV1D_X86_CPU_FLAG_SSE2 = 1 << 1, - DAV1D_X86_CPU_FLAG_SSE3 = 1 << 2, - DAV1D_X86_CPU_FLAG_SSSE3 = 1 << 3, - DAV1D_X86_CPU_FLAG_SSE41 = 1 << 4, - DAV1D_X86_CPU_FLAG_SSE42 = 1 << 5, - DAV1D_X86_CPU_FLAG_AVX = 1 << 6, - DAV1D_X86_CPU_FLAG_AVX2 = 1 << 7, - DAV1D_X86_CPU_FLAG_AVX512 = 1 << 8, /* F + CD + BW + DQ + VL */ + DAV1D_X86_CPU_FLAG_SSE2 = 1 << 0, + DAV1D_X86_CPU_FLAG_SSSE3 = 1 << 1, + DAV1D_X86_CPU_FLAG_SSE41 = 1 << 2, + DAV1D_X86_CPU_FLAG_AVX2 = 1 << 3, + DAV1D_X86_CPU_FLAG_AVX512ICL = 1 << 4, /* F/CD/BW/DQ/VL/VNNI/IFMA/VBMI/VBMI2/ + * VPOPCNTDQ/BITALG/GFNI/VAES/VPCLMULQDQ */ }; unsigned dav1d_get_cpu_flags_x86(void); #endif /* DAV1D_SRC_X86_CPU_H */ diff --git a/third_party/dav1d/src/x86/cpu.c b/third_party/dav1d/src/x86/cpu.c --- a/third_party/dav1d/src/x86/cpu.c +++ b/third_party/dav1d/src/x86/cpu.c @@ -31,41 +31,48 @@ #include "common/attributes.h" #include "src/x86/cpu.h" -void dav1d_cpu_cpuid(uint32_t *info, int leaf); -uint64_t dav1d_cpu_xgetbv(int xcr); +typedef struct { + uint32_t eax, ebx, ecx, edx; +} CpuidRegisters; + +void dav1d_cpu_cpuid(CpuidRegisters *regs, unsigned leaf, unsigned subleaf); +uint64_t dav1d_cpu_xgetbv(unsigned xcr); + +#define X(reg, mask) (((reg) & (mask)) == (mask)) COLD unsigned dav1d_get_cpu_flags_x86(void) { - uint32_t info[4] = {0}, n_ids; + CpuidRegisters r = { 0 }; + dav1d_cpu_cpuid(&r, 0, 0); + const unsigned max_leaf = r.eax; unsigned flags = 0; - dav1d_cpu_cpuid(info, 0); - n_ids = info[0]; - - if (n_ids >= 1) { - dav1d_cpu_cpuid(info, 1); - if (info[3] & (1 << 25)) flags |= DAV1D_X86_CPU_FLAG_SSE; - if (info[3] & (1 << 26)) flags |= DAV1D_X86_CPU_FLAG_SSE2; - if (info[2] & (1 << 0)) flags |= DAV1D_X86_CPU_FLAG_SSE3; - if (info[2] & (1 << 9)) flags |= DAV1D_X86_CPU_FLAG_SSSE3; - if (info[2] & (1 << 19)) flags |= DAV1D_X86_CPU_FLAG_SSE41; - if (info[2] & (1 << 20)) flags |= DAV1D_X86_CPU_FLAG_SSE42; + if (max_leaf >= 1) { + dav1d_cpu_cpuid(&r, 1, 0); + if (X(r.edx, 0x06008000)) /* CMOV/SSE/SSE2 */ { + flags |= DAV1D_X86_CPU_FLAG_SSE2; + if (X(r.ecx, 0x00000201)) /* SSE3/SSSE3 */ { + flags |= DAV1D_X86_CPU_FLAG_SSSE3; + if (X(r.ecx, 0x00080000)) /* SSE4.1 */ + flags |= DAV1D_X86_CPU_FLAG_SSE41; + } + } #if ARCH_X86_64 /* We only support >128-bit SIMD on x86-64. */ - if (info[2] & (1 << 27)) /* OSXSAVE */ { - uint64_t xcr = dav1d_cpu_xgetbv(0); - if ((xcr & 0x00000006) == 0x00000006) /* XMM/YMM */ { - if (info[2] & (1 << 28)) flags |= DAV1D_X86_CPU_FLAG_AVX; - if (n_ids >= 7) { - dav1d_cpu_cpuid(info, 7); - if ((info[1] & 0x00000128) == 0x00000128) + if (X(r.ecx, 0x18000000)) /* OSXSAVE/AVX */ { + const uint64_t xcr0 = dav1d_cpu_xgetbv(0); + if (X(xcr0, 0x00000006)) /* XMM/YMM */ { + if (max_leaf >= 7) { + dav1d_cpu_cpuid(&r, 7, 0); + if (X(r.ebx, 0x00000128)) /* BMI1/BMI2/AVX2 */ { flags |= DAV1D_X86_CPU_FLAG_AVX2; - if ((xcr & 0x000000e0) == 0x000000e0) /* ZMM/OPMASK */ { - if ((info[1] & 0xd0030000) == 0xd0030000) - flags |= DAV1D_X86_CPU_FLAG_AVX512; + if (X(xcr0, 0x000000e0)) /* ZMM/OPMASK */ { + if (X(r.ebx, 0xd0230000) && X(r.ecx, 0x00005f42)) + flags |= DAV1D_X86_CPU_FLAG_AVX512ICL; + } } } } } #endif diff --git a/third_party/dav1d/src/x86/cpuid.asm b/third_party/dav1d/src/x86/cpuid.asm --- a/third_party/dav1d/src/x86/cpuid.asm +++ b/third_party/dav1d/src/x86/cpuid.asm @@ -25,24 +25,24 @@ %include "ext/x86/x86inc.asm" SECTION .text -cglobal cpu_cpuid, 0, 5, 0, info, leaf - mov r4, infomp +cglobal cpu_cpuid, 0, 5, 0, regs, leaf, subleaf + mov r4, regsmp mov eax, leafm - xor ecx, ecx + mov ecx, subleafm %if ARCH_X86_64 - push rbx + mov r5, rbx %endif cpuid mov [r4+4*0], eax mov [r4+4*1], ebx mov [r4+4*2], ecx mov [r4+4*3], edx %if ARCH_X86_64 - pop rbx + mov rbx, r5 %endif RET cglobal cpu_xgetbv, 0, 0, 0, xcr movifnidn ecx, xcrm diff --git a/third_party/dav1d/src/x86/film_grain.asm b/third_party/dav1d/src/x86/film_grain.asm --- a/third_party/dav1d/src/x86/film_grain.asm +++ b/third_party/dav1d/src/x86/film_grain.asm @@ -26,10 +26,12 @@ %include "ext/x86/x86inc.asm" %if ARCH_X86_64 SECTION_RODATA 32 +pb_8x_27_17_8x_17_27: times 8 db 27, 17 + times 8 db 17, 27 pw_1024: times 16 dw 1024 pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0 pw_seed_xor: times 2 dw 0xb524 @@ -42,10 +44,11 @@ mul_bits: dw 256, 128, 64, 32, 16 round_vals: dw 32, 64, 128, 256, 512 max: dw 255, 240, 235 min: dw 0, 16 pb_27_17_17_27: db 27, 17, 17, 27 +pw_1: dw 1 %macro JMP_TABLE 1-* %xdefine %1_table %%table %xdefine %%base %1_table %xdefine %%prefix mangle(private_prefix %+ _%1) @@ -54,12 +57,15 @@ dd %%prefix %+ .ar%2 - %%base %rotate 1 %endrep %endmacro +ALIGN 4 JMP_TABLE generate_grain_y_avx2, 0, 1, 2, 3 JMP_TABLE generate_grain_uv_420_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_avx2, 0, 1, 2, 3 struc FGData .seed: resd 1 .num_y_points: resd 1 .y_points: resb 14 * 2 @@ -67,12 +73,12 @@ .num_uv_points: resd 2 .uv_points: resb 2 * 10 * 2 .scaling_shift: resd 1 .ar_coeff_lag: resd 1 .ar_coeffs_y: resb 24 - .ar_coeffs_uv: resb 2 * 26 ; includes padding - .ar_coeff_shift: resd 1 + .ar_coeffs_uv: resb 2 * 28 ; includes padding + .ar_coeff_shift: resq 1 .grain_scale_shift: resd 1 .uv_mult: resd 2 .uv_luma_mult: resd 2 .uv_offset: resd 2 .overlap_flag: resd 1 @@ -167,13 +173,13 @@ sar val3d, shiftb %endif movsx val0d, byte [bufq+xq] add val3d, val0d cmp val3d, maxd - cmovg val3d, maxd + cmovns val3d, maxd cmp val3d, mind - cmovl val3d, mind + cmovs val3d, mind mov byte [bufq+xq], val3b ; keep val3d in-place as left for next x iteration inc xq jz .x_loop_ar1_end test xq, 3 @@ -188,22 +194,23 @@ RET .ar2: DEFINE_ARGS buf, fg_data, shift mov shiftd, [fg_dataq+FGData.ar_coeff_shift] - movd xm14, [base+hmul_bits-10+shiftq*2] + vpbroadcastw xm14, [base+round_vals-12+shiftq*2] movq xm15, [base+byte_blend+1] pmovsxbw xm8, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7 movd xm9, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11 pmovsxbw xm9, xm9 - DEFINE_ARGS buf, h, x + DEFINE_ARGS buf, fg_data, h, x pshufd xm12, xm9, q0000 pshufd xm13, xm9, q1111 pshufd xm11, xm8, q3333 pshufd xm10, xm8, q2222 pshufd xm9, xm8, q1111 pshufd xm8, xm8, q0000 + pmovzxwd xm14, xm14 sub bufq, 82*73-(82*3+79) mov hd, 70 .y_loop_ar2: mov xq, -76 @@ -231,21 +238,21 @@ pmaddwd xm6, xm10 pmaddwd xm7, xm12 paddd xm4, xm6 paddd xm2, xm7 paddd xm2, xm4 + paddd xm2, xm14 movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] .x_loop_ar2_inner: pmovsxbw xm1, xm0 pmaddwd xm3, xm1, xm13 paddd xm3, xm2 psrldq xm1, 4 ; y=0,x=0 psrldq xm2, 4 ; shift top to next pixel - psrad xm3, 5 - packssdw xm3, xm3 - pmulhrsw xm3, xm14 + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value paddw xm3, xm1 packsswb xm3, xm3 pextrb [bufq+xq], xm3, 0 pslldq xm3, 2 pand xm3, xm15 @@ -272,11 +279,11 @@ %assign stack_size (stack_size+16*12) %else ALLOC_STACK 16*12 %endif mov shiftd, [fg_dataq+FGData.ar_coeff_shift] - movd xm14, [base+hmul_bits-10+shiftq*2] + vpbroadcastw xm14, [base+round_vals-12+shiftq*2] movq xm15, [base+byte_blend] pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-7 pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_y+ 8] ; cf8-15 pmovsxbw xm2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 pshufd xm9, xm0, q1111 @@ -286,14 +293,15 @@ pshufd xm6, xm1, q1111 pshufd xm7, xm1, q2222 pshufd xm8, xm1, q3333 pshufd xm1, xm1, q0000 pshufd xm3, xm2, q1111 + psrldq xm13, xm2, 10 + pinsrw xm2, [pw_1], 5 pshufd xm4, xm2, q2222 - psrldq xm5, xm2, 10 pshufd xm2, xm2, q0000 - pinsrw xm5, [base+round_vals+shiftq*2-10], 3 + pinsrw xm13, [base+round_vals+shiftq*2-10], 3 mova [rsp+ 0*16], xm0 mova [rsp+ 1*16], xm9 mova [rsp+ 2*16], xm10 mova [rsp+ 3*16], xm11 mova [rsp+ 4*16], xm1 @@ -301,13 +309,11 @@ mova [rsp+ 6*16], xm7 mova [rsp+ 7*16], xm8 mova [rsp+ 8*16], xm2 mova [rsp+ 9*16], xm3 mova [rsp+10*16], xm4 - mova [rsp+11*16], xm5 - pxor xm13, xm13 - DEFINE_ARGS buf, h, x + DEFINE_ARGS buf, fg_data, h, x sub bufq, 82*73-(82*3+79) mov hd, 70 .y_loop_ar3: mov xq, -76 @@ -372,29 +378,28 @@ palignr xm9, xm5, xm2, 10 palignr xm5, xm5, xm2, 12 punpcklwd xm6, xm7 punpcklwd xm8, xm9 - punpcklwd xm5, xm13 + punpcklwd xm5, xm14 pmaddwd xm6, [rsp+ 8*16] pmaddwd xm8, [rsp+ 9*16] pmaddwd xm5, [rsp+10*16] paddd xm0, xm6 paddd xm8, xm5 paddd xm0, xm8 movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] .x_loop_ar3_inner: pmovsxbw xm2, xm1 - pmaddwd xm2, [rsp+16*11] + pmaddwd xm2, xm13 pshufd xm3, xm2, q1111 paddd xm2, xm3 ; left+cur paddd xm2, xm0 ; add top psrldq xm0, 4 - psrad xm2, 5 - packssdw xm2, xm2 - pmulhrsw xm2, xm14 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value packsswb xm2, xm2 pextrb [bufq+xq], xm2, 0 pslldq xm2, 3 pand xm2, xm15 pandn xm1, xm15, xm1 @@ -410,12 +415,13 @@ add bufq, 82 dec hd jg .y_loop_ar3 RET +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y INIT_XMM avx2 -cglobal generate_grain_uv_420, 4, 10, 16, buf, bufy, fg_data, uv +cglobal generate_grain_uv_%1, 4, 10, 16, buf, bufy, fg_data, uv lea r4, [pb_mask] %define base r4-pb_mask movq xm1, [base+rnd_next_upperbit_mask] movq xm4, [base+mul_bits] movq xm7, [base+hmul_bits] @@ -425,15 +431,21 @@ vpbroadcastw xm0, [fg_dataq+FGData.seed] vpbroadcastw xm9, [base+pw_seed_xor+uvq*4] pxor xm0, xm9 vpbroadcastd xm9, [base+pd_m65536] lea r6, [gaussian_sequence] - mov r7d, 38 +%if %2 + mov r7d, 73-35*%3 add bufq, 44 .loop_y: mov r5, -44 .loop_x: +%else + mov r5, -73*82 + sub bufq, r5 +.loop: +%endif pand xm2, xm0, xm1 psrlw xm3, xm2, 10 por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set pmullw xm2, xm4 ; bits 0x0f00 are set pshufb xm2, xm5, xm2 ; set 15th bit for next 4 seeds @@ -452,125 +464,208 @@ packusdw xm2, xm2 pmulhrsw xm2, xm8 packsswb xm2, xm2 movd [bufq+r5], xm2 add r5, 4 +%if %2 jl .loop_x add bufq, 82 dec r7d jg .loop_y +%else + jl .loop +%endif ; auto-regression code movsxd r5, [fg_dataq+FGData.ar_coeff_lag] - movsxd r5, [base+generate_grain_uv_420_avx2_table+r5*4] - lea r5, [r5+base+generate_grain_uv_420_avx2_table] + movsxd r5, [base+generate_grain_uv_%1_avx2_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_avx2_table] jmp r5 .ar0: INIT_YMM avx2 DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift - imul uvd, 25 + imul uvd, 28 mov shiftd, [fg_dataq+FGData.ar_coeff_shift] movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] movd xm3, [base+hmul_bits+shiftq*2] DEFINE_ARGS buf, bufy, h pmovsxbw xm4, xm4 +%if %2 vpbroadcastd m7, [pb_1] - vpbroadcastw m6, [hmul_bits+4] + vpbroadcastw m6, [hmul_bits+2+%3*2] +%endif vpbroadcastw m4, xm4 vpbroadcastw m3, xm3 - sub bufq, 82*38+82-(82*3+41) + pxor m12, m12 +%if %2 + sub bufq, 82*(73-35*%3)+82-(82*3+41) +%else + sub bufq, 82*70-3 +%endif add bufyq, 3+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar0: +%if %2 ; first 32 pixels movu xm8, [bufyq] +%if %3 movu xm9, [bufyq+82] +%endif movu xm10, [bufyq+16] +%if %3 movu xm11, [bufyq+82+16] +%endif vinserti128 m8, [bufyq+32], 1 +%if %3 vinserti128 m9, [bufyq+82+32], 1 +%endif vinserti128 m10, [bufyq+48], 1 +%if %3 vinserti128 m11, [bufyq+82+48], 1 +%endif pmaddubsw m8, m7, m8 +%if %3 pmaddubsw m9, m7, m9 +%endif pmaddubsw m10, m7, m10 +%if %3 pmaddubsw m11, m7, m11 paddw m8, m9 paddw m10, m11 +%endif pmulhrsw m8, m6 pmulhrsw m10, m6 +%else + xor r3d, r3d + ; first 32x2 pixels +.x_loop_ar0: + movu m8, [bufyq+r3] + pcmpgtb m9, m12, m8 + punpckhbw m10, m8, m9 + punpcklbw m8, m9 +%endif pmullw m8, m4 pmullw m10, m4 pmulhrsw m8, m3 pmulhrsw m10, m3 - packsswb m8, m10 +%if %2 movu m0, [bufq] - punpckhbw m1, m0, m8 - punpcklbw m0, m8 - pmaddubsw m1, m7, m1 - pmaddubsw m0, m7, m0 - packsswb m0, m1 +%else + movu m0, [bufq+r3] +%endif + pcmpgtb m1, m12, m0 + punpckhbw m9, m0, m1 + punpcklbw m0, m1 + paddw m0, m8 + paddw m9, m10 + packsswb m0, m9 +%if %2 movu [bufq], m0 +%else + movu [bufq+r3], m0 + add r3d, 32 + cmp r3d, 64 + jl .x_loop_ar0 +%endif - ; last 6 pixels + ; last 6/12 pixels movu xm8, [bufyq+32*2] +%if %2 +%if %3 movu xm9, [bufyq+32*2+82] +%endif pmaddubsw xm8, xm7, xm8 +%if %3 pmaddubsw xm9, xm7, xm9 paddw xm8, xm9 +%endif pmulhrsw xm8, xm6 pmullw xm8, xm4 pmulhrsw xm8, xm3 - packsswb xm8, xm8 movq xm0, [bufq+32] - punpcklbw xm8, xm0 - pmaddubsw xm8, xm7, xm8 + pcmpgtb xm9, xm12, xm0 + punpcklbw xm9, xm0, xm9 + paddw xm8, xm9 packsswb xm8, xm8 vpblendw xm0, xm8, xm0, 1000b movq [bufq+32], xm0 +%else + pcmpgtb xm9, xm12, xm8 + punpckhbw xm10, xm8, xm9 + punpcklbw xm8, xm9 + pmullw xm10, xm4 + pmullw xm8, xm4 + pmulhrsw xm10, xm3 + pmulhrsw xm8, xm3 + movu xm0, [bufq+64] + pcmpgtb xm9, xm12, xm0 + punpcklbw xm1, xm0, xm9 + punpckhbw xm9, xm0, xm9 + paddw xm1, xm8 + paddw xm9, xm10 + packsswb xm1, xm9 + vpblendw xm0, xm1, xm0, 11000000b + movu [bufq+64], xm0 +%endif add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar0 RET .ar1: INIT_XMM avx2 DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x, shift - imul uvd, 25 + imul uvd, 28 mov shiftd, [fg_dataq+FGData.ar_coeff_shift] movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] pinsrb xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 3 DEFINE_ARGS buf, bufy, h, val0, val3, cf3, min, max, x, shift pmovsxbw xm4, xm4 pshufd xm5, xm4, q1111 pshufd xm4, xm4, q0000 pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd +%if %2 vpbroadcastd xm7, [pb_1] - vpbroadcastw xm6, [hmul_bits+4] + vpbroadcastw xm6, [hmul_bits+2+%3*2] +%endif vpbroadcastd xm3, xm3 - sub bufq, 82*38+44-(82*3+41) +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 mov mind, -128 mov maxd, 127 .y_loop_ar1: - mov xq, -38 + mov xq, -(76>>%2) movsx val3d, byte [bufq+xq-1] .x_loop_ar1: pmovsxbw xm0, [bufq+xq-82-1] ; top/left +%if %2 movq xm8, [bufyq+xq*2] +%if %3 movq xm9, [bufyq+xq*2+82] +%endif +%endif psrldq xm2, xm0, 2 ; top psrldq xm1, xm0, 4 ; top/right +%if %2 pmaddubsw xm8, xm7, xm8 +%if %3 pmaddubsw xm9, xm7, xm9 paddw xm8, xm9 +%endif pmulhrsw xm8, xm6 +%else + pmovsxbw xm8, [bufyq+xq] +%endif punpcklwd xm0, xm2 punpcklwd xm1, xm8 pmaddwd xm0, xm4 pmaddwd xm1, xm5 paddd xm0, xm1 @@ -582,52 +677,57 @@ add val3d, val0d sarx val3d, val3d, shiftd movsx val0d, byte [bufq+xq] add val3d, val0d cmp val3d, maxd - cmovg val3d, maxd + cmovns val3d, maxd cmp val3d, mind - cmovl val3d, mind + cmovs val3d, mind mov byte [bufq+xq], val3b ; keep val3d in-place as left for next x iteration inc xq jz .x_loop_ar1_end test xq, 3 jnz .x_loop_ar1_inner jmp .x_loop_ar1 .x_loop_ar1_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar1 RET .ar2: DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift mov shiftd, [fg_dataq+FGData.ar_coeff_shift] - imul uvd, 25 - movd xm15, [base+hmul_bits-10+shiftq*2] + imul uvd, 28 + vpbroadcastw xm15, [base+round_vals-12+shiftq*2] pmovsxbw xm8, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-7 pmovsxbw xm9, [fg_dataq+FGData.ar_coeffs_uv+uvq+8] ; cf8-12 - vpbroadcastw xm7, [base+hmul_bits+4] + pinsrw xm9, [base+pw_1], 5 +%if %2 + vpbroadcastw xm7, [base+hmul_bits+2+%3*2] vpbroadcastd xm6, [base+pb_1] - DEFINE_ARGS buf, bufy, h, x +%endif + DEFINE_ARGS buf, bufy, fg_data, h, unused, x pshufd xm12, xm9, q0000 pshufd xm13, xm9, q1111 pshufd xm14, xm9, q2222 - pxor xm10, xm10 - vpblendw xm14, xm10, 10101010b pshufd xm11, xm8, q3333 pshufd xm10, xm8, q2222 pshufd xm9, xm8, q1111 pshufd xm8, xm8, q0000 - sub bufq, 82*38+44-(82*3+41) +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar2: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar2: pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] psrldq xm2, xm0, 2 ; y=-2,x=[-1,+5] @@ -652,29 +752,35 @@ pmaddwd xm3, xm12 paddd xm4, xm0 paddd xm2, xm3 paddd xm2, xm4 +%if %2 movq xm0, [bufyq+xq*2] +%if %3 movq xm3, [bufyq+xq*2+82] +%endif pmaddubsw xm0, xm6, xm0 +%if %3 pmaddubsw xm3, xm6, xm3 paddw xm0, xm3 +%endif pmulhrsw xm0, xm7 - punpcklwd xm0, xm0 +%else + pmovsxbw xm0, [bufyq+xq] +%endif + punpcklwd xm0, xm15 pmaddwd xm0, xm14 paddd xm2, xm0 movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] .x_loop_ar2_inner: pmovsxbw xm0, xm0 pmaddwd xm3, xm0, xm13 paddd xm3, xm2 psrldq xm2, 4 ; shift top to next pixel - psrad xm3, 5 - packssdw xm3, xm3 - pmulhrsw xm3, xm15 + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] pslldq xm3, 2 psrldq xm0, 2 paddw xm3, xm0 vpblendw xm0, xm3, 00000010b packsswb xm0, xm0 @@ -685,23 +791,23 @@ jnz .x_loop_ar2_inner jmp .x_loop_ar2 .x_loop_ar2_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar2 RET .ar3: DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift SUB rsp, 16*12 %assign stack_size_padded (stack_size_padded+16*12) %assign stack_size (stack_size+16*12) mov shiftd, [fg_dataq+FGData.ar_coeff_shift] - imul uvd, 25 - movd xm14, [base+hmul_bits-10+shiftq*2] + imul uvd, 28 + vpbroadcastw xm14, [base+round_vals-12+shiftq*2] pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-7 pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 8] ; cf8-15 pmovsxbw xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-23 pmovsxbw xm5, [fg_dataq+FGData.ar_coeffs_uv+uvq+24] ; cf24 [luma] pshufd xm9, xm0, q1111 @@ -717,10 +823,11 @@ vpbroadcastw xm5, xm5 vpblendw xm4, xm5, 10101010b ; interleave luma cf psrldq xm5, xm2, 10 pshufd xm2, xm2, q0000 pinsrw xm5, [base+round_vals+shiftq*2-10], 3 + pmovzxwd xm14, xm14 mova [rsp+ 0*16], xm0 mova [rsp+ 1*16], xm9 mova [rsp+ 2*16], xm10 mova [rsp+ 3*16], xm11 mova [rsp+ 4*16], xm1 @@ -729,18 +836,24 @@ mova [rsp+ 7*16], xm8 mova [rsp+ 8*16], xm2 mova [rsp+ 9*16], xm3 mova [rsp+10*16], xm4 mova [rsp+11*16], xm5 +%if %2 vpbroadcastd xm13, [base+pb_1] - vpbroadcastw xm15, [base+hmul_bits+4] - DEFINE_ARGS buf, bufy, h, x - sub bufq, 82*38+44-(82*3+41) + vpbroadcastw xm15, [base+hmul_bits+2+%3*2] +%endif + DEFINE_ARGS buf, bufy, fg_data, h, unused, x +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar3: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar3: movu xm0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] movu xm1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] movu xm2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] @@ -799,38 +912,46 @@ psrldq xm7, xm2, 6 psrldq xm8, xm2, 8 palignr xm9, xm5, xm2, 10 palignr xm5, xm5, xm2, 12 +%if %2 movq xm1, [bufyq+xq*2] +%if %3 movq xm2, [bufyq+xq*2+82] +%endif pmaddubsw xm1, xm13, xm1 +%if %3 pmaddubsw xm2, xm13, xm2 paddw xm1, xm2 +%endif pmulhrsw xm1, xm15 +%else + pmovsxbw xm1, [bufyq+xq] +%endif punpcklwd xm6, xm7 punpcklwd xm8, xm9 punpcklwd xm5, xm1 pmaddwd xm6, [rsp+ 8*16] pmaddwd xm8, [rsp+ 9*16] pmaddwd xm5, [rsp+10*16] paddd xm0, xm6 paddd xm8, xm5 paddd xm0, xm8 + paddd xm0, xm14 movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] .x_loop_ar3_inner: pmovsxbw xm1, xm1 pmaddwd xm2, xm1, [rsp+16*11] pshufd xm3, xm2, q1111 paddd xm2, xm3 ; left+cur paddd xm2, xm0 ; add top psrldq xm0, 4 - psrad xm2, 5 - packssdw xm2, xm2 - pmulhrsw xm2, xm14 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw, we only care about one value pslldq xm2, 6 vpblendw xm1, xm2, 1000b packsswb xm1, xm1 pextrb [bufq+xq], xm1, 3 psrldq xm1, 1 @@ -840,14 +961,19 @@ jnz .x_loop_ar3_inner jmp .x_loop_ar3 .x_loop_ar3_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar3 RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 INIT_YMM avx2 cglobal fgy_32x32xn, 6, 13, 16, dst, src, stride, fg_data, w, scaling, grain_lut pcmpeqw m10, m10 psrld m10, 24 @@ -1187,13 +1313,12 @@ add grain_lutq, 82 dec hw jz .end_y_v_overlap ; 2 lines get vertical overlap, then fall back to non-overlap code for ; remaining (up to) 30 lines - xor hd, 0x10000 - test hd, 0x10000 - jnz .loop_y_v_overlap + btc hd, 16 + jnc .loop_y_v_overlap jmp .loop_y .end_y_v_overlap: add wq, 32 jge .end_hv @@ -1320,25 +1445,25 @@ add grain_lutq, 82 dec hw jz .end_y_hv_overlap ; 2 lines get vertical overlap, then fall back to non-overlap code for ; remaining (up to) 30 lines - xor hd, 0x10000 - test hd, 0x10000 - jnz .loop_y_hv_overlap + btc hd, 16 + jnc .loop_y_hv_overlap jmp .loop_y_h_overlap .end_y_hv_overlap: add wq, 32 lea srcq, [src_bakq+wq] jl .loop_x_hv_overlap .end_hv: RET -cglobal fguv_32x32xn_i420, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ - grain_lut, h, sby, luma, lstride, uv_pl, is_id +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id pcmpeqw m10, m10 psrld m10, 24 mov r7d, [fg_dataq+FGData.scaling_shift] lea r8, [pb_mask] %define base r8-pb_mask @@ -1350,22 +1475,26 @@ vpbroadcastw m12, [base+max+r7*2] cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 jne .csfl -%macro FGUV_32x32xN_LOOP 1 ; not-csfl +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap %if %1 mov r7d, dword r11m vpbroadcastb m0, [fg_dataq+FGData.uv_mult+r7*4] vpbroadcastb m1, [fg_dataq+FGData.uv_luma_mult+r7*4] punpcklbw m14, m1, m0 vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r7*4] %else vpbroadcastd m14, [pw_1024] +%if %2 vpbroadcastd m15, [pb_23_22] +%else + vpbroadcastd xm15, [pb_27_17_17_27] +%endif %endif mov overlapd, [fg_dataq+FGData.overlap_flag] movifnidn sbyd, sbym test sbyd, sbyd @@ -1383,11 +1512,11 @@ unused2, unused3, see, overlap, unused4, unused5, lstride mov lumaq, r9mp lea r12, [srcq+wq] lea r13, [dstq+wq] - lea r14, [lumaq+wq*2] + lea r14, [lumaq+wq*(1+%2)] mov r11mp, r12 mov r12mp, r13 mov lstrideq, r10mp neg wq @@ -1404,35 +1533,43 @@ mov offxd, seed rorx offyd, seed, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, overlap, unused1, unused2, lstride mov hd, hm mov grain_lutq, grain_lutmp %%loop_y: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + pxor m2, m2 + mova m4, [lumaq] + mova m0, [srcq] +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1440,10 +1577,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1468,12 +1608,16 @@ ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; grain = grain_lut[offy+y][offx+x] +%if %2 movu xm3, [grain_lutq+offxyq+ 0] vinserti128 m3, [grain_lutq+offxyq+82], 1 +%else + movu m3, [grain_lutq+offxyq] +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[luma_src] * grain, scaling_shift) @@ -1488,25 +1632,35 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 +%if %2 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 - sub hb, 2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 jg %%loop_y - add wq, 16 + add wq, 32>>%2 jge %%end mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq test overlapd, overlapd jz %%loop_x @@ -1524,40 +1678,48 @@ cmovp seed, r6d ; updated seed DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ offx, offy, see, left_offxy, unused1, unused2, lstride - lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx + lea left_offxyd, [offyd+(32>>%2)] ; previous column's offy*stride+offx mov offxd, seed rorx offyd, seed, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, left_offxy, unused1, unused2, lstride mov hd, hm mov grain_lutq, grain_lutmp %%loop_y_h_overlap: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1565,10 +1727,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1593,10 +1758,11 @@ ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; grain = grain_lut[offy+y][offx+x] +%if %2 %if %1 vpbroadcastd m6, [pb_23_22] ; FIXME %endif movu xm3, [grain_lutq+offxyq+ 0] movd xm4, [grain_lutq+left_offxyq+ 0] @@ -1612,10 +1778,29 @@ %endif packsswb m4, m4 pcmpeqw m6, m6 ; FIXME psrldq m6, 15 ; FIXME vpblendvb m3, m3, m4, m6 +%else +%if %1 + vpbroadcastd xm6, [pb_27_17_17_27] +%endif + movu m3, [grain_lutq+offxyq] + movd xm4, [grain_lutq+left_offxyq] + punpcklbw xm4, xm3 +%if %1 + pmaddubsw xm4, xm6, xm4 + pmulhrsw xm4, [pw_1024] +%else + pmaddubsw xm4, xm15, xm4 + pmulhrsw xm4, xm14 +%endif + packsswb xm4, xm4 + pcmpeqw xm6, xm6 + psrldq xm6, 14 + vpblendvb m3, m3, m4, m6 +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[luma_src] * grain, scaling_shift) @@ -1630,25 +1815,35 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 +%if %2 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 - sub hb, 2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(1+%2) + sub hb, 1+%2 jg %%loop_y_h_overlap - add wq, 16 + add wq, 32>>%2 jge %%end mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq ; r8m = sbym cmp dword r8m, 0 @@ -1677,11 +1872,11 @@ unused1, unused2, see, overlap, unused3, unused4, lstride mov lumaq, r9mp lea r12, [srcq+wq] lea r13, [dstq+wq] - lea r14, [lumaq+wq*2] + lea r14, [lumaq+wq*(1+%2)] mov r11mp, r12 mov r12mp, r13 mov lstrideq, r10mp neg wq @@ -1704,39 +1899,50 @@ rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, overlap, top_offxy, unused, lstride movzx top_offxyd, offxyw shr offxyd, 16 mov hd, hm mov grain_lutq, grain_lutmp +%if %2 == 0 + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27] +%endif %%loop_y_v_overlap: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1744,10 +1950,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1767,15 +1976,46 @@ pand m5, m10 pand m6, m10 packusdw m8, m4 packusdw m5, m6 +%if %2 ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word +%endif ; grain = grain_lut[offy+y][offx+x] +%if %3 == 0 +%if %2 + mova m6, [pb_8x_27_17_8x_17_27] + movu xm3, [grain_lutq+offxyq] + movu xm4, [grain_lutq+top_offxyq] + vinserti128 m3, [grain_lutq+offxyq+82], 1 + vinserti128 m4, [grain_lutq+top_offxyq+82], 1 +%else + movu m3, [grain_lutq+offxyq] + movu m4, [grain_lutq+top_offxyq] +%endif + punpckhbw m9, m4, m3 + punpcklbw m4, m3 +%if %2 + pmaddubsw m9, m6, m9 + pmaddubsw m4, m6, m4 +%else + pmaddubsw m9, m1, m9 + pmaddubsw m4, m1, m4 +%endif +%if %1 + pmulhrsw m9, [pw_1024] + pmulhrsw m4, [pw_1024] +%else + pmulhrsw m9, m14 + pmulhrsw m4, m14 +%endif + packsswb m3, m4, m9 +%else %if %1 vpbroadcastd m6, [pb_23_22] %endif movq xm3, [grain_lutq+offxyq] movq xm4, [grain_lutq+top_offxyq] @@ -1791,10 +2031,11 @@ %endif packsswb m4, m4 vpermq m4, m4, q3120 ; only interpolate first line, insert second line unmodified vinserti128 m3, m4, [grain_lutq+offxyq+82], 1 +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[luma_src] * grain, scaling_shift) @@ -1802,34 +2043,60 @@ pmullw m3, m5 pmulhrsw m2, m11 pmulhrsw m3, m11 ; dst = clip_pixel(src, noise) +%if %2 paddw m0, m2 paddw m1, m3 pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + pxor m6, m6 + punpckhbw m9, m0, m6 + punpcklbw m0, m6 ; m0-1: src as word - sub hb, 2 + paddw m0, m2 + paddw m9, m3 + pmaxsw m0, m13 + pmaxsw m9, m13 + pminsw m0, m12 + pminsw m9, m12 + packuswb m0, m9 + mova [dstq], m0 +%endif + + sub hb, 1+%2 jl %%end_y_v_overlap +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 +%if %2 == 0 + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27+16] + btc hd, 16 + jnc %%loop_y_v_overlap +%endif jmp %%loop_y %%end_y_v_overlap: - add wq, 16 + add wq, 32>>%2 jge %%end_hv mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to @@ -1850,45 +2117,56 @@ rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride - lea topleft_offxyq, [top_offxyq+16] - lea left_offxyq, [offyq+16] + lea topleft_offxyq, [top_offxyq+(32>>%2)] + lea left_offxyq, [offyq+(32>>%2)] rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride movzx top_offxyd, offxyw shr offxyd, 16 mov hd, hm mov grain_lutq, grain_lutmp +%if %2 == 0 + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27] +%endif %%loop_y_hv_overlap: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1896,10 +2174,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1919,48 +2200,98 @@ pand m5, m10 pand m6, m10 packusdw m8, m4 packusdw m5, m6 +%if %2 ; unpack chroma source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word +%endif ; grain = grain_lut[offy+y][offx+x] %if %1 +%if %2 vpbroadcastd m9, [pb_23_22] +%else + vpbroadcastd xm9, [pb_27_17_17_27] %endif +%endif + +%if %2 movu xm3, [grain_lutq+offxyq] +%if %3 movq xm6, [grain_lutq+top_offxyq] +%else + movu xm6, [grain_lutq+top_offxyq] +%endif vinserti128 m3, [grain_lutq+offxyq+82], 1 +%if %3 vinserti128 m6, [grain_lutq+top_offxyq+8], 1 +%else + vinserti128 m6, [grain_lutq+top_offxyq+82], 1 +%endif +%else + movu m3, [grain_lutq+offxyq] + movu m6, [grain_lutq+top_offxyq] +%endif movd xm4, [grain_lutq+left_offxyq] movd xm7, [grain_lutq+topleft_offxyq] +%if %2 vinserti128 m4, [grain_lutq+left_offxyq+82], 1 +%if %3 == 0 + vinserti128 m7, [grain_lutq+topleft_offxyq+82], 1 +%endif +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) +%if %2 punpcklbw m4, m3 +%if %3 punpcklbw xm7, xm6 +%else + punpcklbw m7, m6 +%endif + punpcklwd m4, m7 %if %1 pmaddubsw m4, m9, m4 - pmaddubsw xm7, xm9, xm7 pmulhrsw m4, [pw_1024] - pmulhrsw xm7, [pw_1024] %else pmaddubsw m4, m15, m4 - pmaddubsw xm7, xm15, xm7 pmulhrsw m4, m14 - pmulhrsw xm7, xm14 %endif packsswb m4, m4 - packsswb xm7, xm7 pcmpeqw m9, m9 ; this is kind of ugly psrldq m9, 15 vpblendvb m3, m3, m4, m9 - shufpd m9, m9, m9, 1110b - vpblendvb m6, m6, m7, m9 + psrldq m4, 1 +%if %3 + shufpd m9, m9, m9, 1110b ; clear upper lane +%endif + vpblendvb m6, m6, m4, m9 +%else + punpcklbw xm4, xm3 + punpcklbw xm7, xm6 + punpckldq xm4, xm7 +%if %1 + pmaddubsw xm4, xm9, xm4 + pmulhrsw xm4, [pw_1024] +%else + pmaddubsw xm4, xm15, xm4 + pmulhrsw xm4, xm14 +%endif + packsswb xm4, xm4 + pcmpeqw xm9, xm9 ; this is kind of ugly + psrldq xm9, 14 + vpblendvb m3, m3, m4, m9 + psrldq xm4, 2 + vpblendvb m6, m6, m4, m9 +%endif + + ; followed by v interpolation (top | cur -> cur) +%if %3 vpermq m9, m3, q3120 - ; followed by v interpolation (top | cur -> cur) punpcklbw m6, m9 %if %1 vpbroadcastd m9, [pb_23_22] pmaddubsw m6, m9, m6 pmulhrsw m6, [pw_1024] @@ -1969,10 +2300,30 @@ pmulhrsw m6, m14 %endif packsswb m6, m6 vpermq m6, m6, q3120 vpblendd m3, m3, m6, 00001111b +%else + punpckhbw m9, m6, m3 + punpcklbw m6, m3 +%if %2 + mova m3, [pb_8x_27_17_8x_17_27] + pmaddubsw m9, m3, m9 + pmaddubsw m6, m3, m6 +%else + pmaddubsw m9, m1, m9 + pmaddubsw m6, m1, m6 +%endif +%if %1 + pmulhrsw m9, [pw_1024] + pmulhrsw m6, [pw_1024] +%else + pmulhrsw m9, m14 + pmulhrsw m6, m14 +%endif + packsswb m3, m6, m9 +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[src] * grain, scaling_shift) @@ -1980,41 +2331,74 @@ pmullw m3, m5 pmulhrsw m2, m11 pmulhrsw m3, m11 ; dst = clip_pixel(src, noise) +%if %2 paddw m0, m2 paddw m1, m3 pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + pxor m6, m6 + punpckhbw m9, m0, m6 + punpcklbw m0, m6 ; m0-1: src as word + paddw m0, m2 + paddw m9, m3 + pmaxsw m0, m13 + pmaxsw m9, m13 + pminsw m0, m12 + pminsw m9, m12 + packuswb m0, m9 + mova [dstq], m0 +%endif +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 - sub hb, 2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 +%if %2 jg %%loop_y_h_overlap +%else + je %%end_y_hv_overlap + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27+16] + btc hd, 16 + jnc %%loop_y_hv_overlap + jmp %%loop_y_h_overlap +%endif %%end_y_hv_overlap: - add wq, 16 + add wq, 32>>%2 jge %%end_hv mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq jmp %%loop_x_hv_overlap %%end_hv: RET %endmacro - FGUV_32x32xN_LOOP 1 + %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: - FGUV_32x32xN_LOOP 0 + %%FGUV_32x32xN_LOOP 0, %2, %3 +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 %endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/film_grain_init_tmpl.c b/third_party/dav1d/src/x86/film_grain_init_tmpl.c --- a/third_party/dav1d/src/x86/film_grain_init_tmpl.c +++ b/third_party/dav1d/src/x86/film_grain_init_tmpl.c @@ -26,22 +26,52 @@ */ #include "src/cpu.h" #include "src/film_grain.h" +decl_generate_grain_y_fn(dav1d_generate_grain_y_ssse3); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_ssse3); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_422_ssse3); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_444_ssse3); +decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_ssse3); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_ssse3); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i422_ssse3); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i444_ssse3); + decl_generate_grain_y_fn(dav1d_generate_grain_y_avx2); decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_avx2); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_422_avx2); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_444_avx2); decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_avx2); decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_avx2); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i422_avx2); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i444_avx2); COLD void bitfn(dav1d_film_grain_dsp_init_x86)(Dav1dFilmGrainDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + +#if BITDEPTH == 8 + c->generate_grain_y = dav1d_generate_grain_y_ssse3; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_generate_grain_uv_420_ssse3; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_generate_grain_uv_422_ssse3; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_generate_grain_uv_444_ssse3; + c->fgy_32x32xn = dav1d_fgy_32x32xn_ssse3; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_fguv_32x32xn_i420_ssse3; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_fguv_32x32xn_i422_ssse3; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_fguv_32x32xn_i444_ssse3; +#endif + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; #if BITDEPTH == 8 && ARCH_X86_64 c->generate_grain_y = dav1d_generate_grain_y_avx2; c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_generate_grain_uv_420_avx2; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_generate_grain_uv_422_avx2; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_generate_grain_uv_444_avx2; c->fgy_32x32xn = dav1d_fgy_32x32xn_avx2; c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_fguv_32x32xn_i420_avx2; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_fguv_32x32xn_i422_avx2; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_fguv_32x32xn_i444_avx2; #endif } diff --git a/third_party/dav1d/src/x86/film_grain_ssse3.asm b/third_party/dav1d/src/x86/film_grain_ssse3.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/film_grain_ssse3.asm @@ -0,0 +1,3300 @@ +; Copyright © 2019, VideoLAN and dav1d authors +; Copyright © 2019, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +SECTION_RODATA + +pw_1024: times 8 dw 1024 +pb_27_17: times 8 db 27, 17 +pb_17_27: times 8 db 17, 27 +pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 +rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 +byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0 +pw_seed_xor: times 2 dw 0xb524 + times 2 dw 0x49d8 +pb_23_22: times 2 db 23, 22 +pb_1: times 4 db 1 +hmul_bits: dw 32768, 16384, 8192, 4096 +round: dw 2048, 1024, 512 +mul_bits: dw 256, 128, 64, 32, 16 +round_vals: dw 32, 64, 128, 256, 512 +max: dw 255, 240, 235 +min: dw 0, 16 +pw_1: dw 1 + +%define pb_27_17_17_27 pb_17_27 - 2 + +%macro JMP_TABLE 1-* + %xdefine %1_table %%table + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + %%table: + %rep %0 - 1 + dd %%prefix %+ .ar%2 - %%base + %rotate 1 + %endrep +%endmacro + +JMP_TABLE generate_grain_y_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_ssse3, 0, 1, 2, 3 + +struc FGData + .seed: resd 1 + .num_y_points: resd 1 + .y_points: resb 14 * 2 + .chroma_scaling_from_luma: resd 1 + .num_uv_points: resd 2 + .uv_points: resb 2 * 10 * 2 + .scaling_shift: resd 1 + .ar_coeff_lag: resd 1 + .ar_coeffs_y: resb 24 + .ar_coeffs_uv: resb 2 * 28 ; includes padding + .ar_coeff_shift: resq 1 + .grain_scale_shift: resd 1 + .uv_mult: resd 2 + .uv_luma_mult: resd 2 + .uv_offset: resd 2 + .overlap_flag: resd 1 + .clip_to_restricted_range: resd 1 +endstruc + +cextern gaussian_sequence + +SECTION .text + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +INIT_XMM ssse3 +cglobal generate_grain_y, 2, 7 + 2 * ARCH_X86_64, 16, buf, fg_data + LEA r4, $$ +%define base r4-$$ + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r2d, [fg_dataq+FGData.grain_scale_shift] + movd m2, [base+round+r2*2] + movd m0, [fg_dataq+FGData.seed] + mova m5, [base+pb_mask] + pshuflw m2, m2, q0000 + pshuflw m0, m0, q0000 + mov r2, -73*82 + sub bufq, r2 + lea r3, [base+gaussian_sequence] +.loop: + pand m6, m0, m1 + psrlw m3, m6, 10 + por m6, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m6, m4 ; bits 0x0f00 are set + pshufb m3, m5, m6 ; set 15th bit for next 4 seeds + psllq m6, m3, 30 + por m3, m6 + psllq m6, m3, 15 + por m3, m6 ; aggregate each bit into next seed's high bit + pmulhuw m6, m0, m7 + por m3, m6 ; 4 next output seeds + pshuflw m0, m3, q3333 + psrlw m3, 5 +%if ARCH_X86_64 + movq r6, m3 + mov r8, r6 + movzx r5d, r6w + shr r6d, 16 + shr r8, 32 + movzx r7, r8w + shr r8, 16 + + movd m6, [r3+r5*2] + pinsrw m6, [r3+r6*2], 1 + pinsrw m6, [r3+r7*2], 2 + pinsrw m6, [r3+r8*2], 3 +%else + movd r6, m3 + pshuflw m3, m3, q3232 + movzx r5, r6w + shr r6, 16 + + movd m6, [r3+r5*2] + pinsrw m6, [r3+r6*2], 1 + + movd r6, m3 + movzx r5, r6w + shr r6, 16 + + pinsrw m6, [r3+r5*2], 2 + pinsrw m6, [r3+r6*2], 3 +%endif + pmulhrsw m6, m2 + packsswb m6, m6 + movd [bufq+r2], m6 + add r2, 4 + jl .loop + + ; auto-regression code + movsxd r2, [fg_dataq+FGData.ar_coeff_lag] + movsxd r2, [base+generate_grain_y_ssse3_table+r2*4] + lea r2, [r2+base+generate_grain_y_ssse3_table] + jmp r2 + +.ar1: +%if ARCH_X86_32 + DEFINE_ARGS buf, fg_data, cf3, unused, val3, min, max +%elif WIN64 + DEFINE_ARGS shift, fg_data, cf3, buf, val3, min, max, x, val0 + mov bufq, r0 +%else + DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0 +%endif + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] + movd m4, [fg_dataq+FGData.ar_coeffs_y] + mov ecx, [fg_dataq+FGData.ar_coeff_shift] +%if ARCH_X86_32 + mov r1m, cf3d + DEFINE_ARGS buf, shift, val3, min, max, x, val0 +%define hd r0mp +%define cf3d r1mp +%elif WIN64 + DEFINE_ARGS shift, h, cf3, buf, val3, min, max, x, val0 +%else + DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0 +%endif + pxor m6, m6 + pcmpgtb m7, m6, m4 + punpcklbw m4, m7 + pinsrw m4, [base+pw_1], 3 + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + movd m3, [base+round_vals+shiftq*2-12] ; rnd + pshuflw m3, m3, q0000 + sub bufq, 82*73-(82*3+79) + mov hd, 70 + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -76 + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: + movq m0, [bufq+xq-82-1] ; top/left + pcmpgtb m7, m6, m0 + punpcklbw m0, m7 + psrldq m2, m0, 2 ; top + psrldq m1, m0, 4 ; top/right + punpcklwd m0, m2 + punpcklwd m1, m3 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 + imul val3d, cf3d + add val3d, val0d + sar val3d, shiftb + movsx val0d, byte [bufq+xq] + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov byte [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82 + dec hd + jg .y_loop_ar1 +.ar0: + RET + +.ar2: +%if ARCH_X86_32 +%assign stack_offset_old stack_offset + ALLOC_STACK -16*8 +%endif + DEFINE_ARGS buf, fg_data, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m6, [base+round_vals-12+shiftq*2] + movd m7, [base+byte_blend+1] + SCRATCH 7, 15, 7 + movq m0, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7 + movd m1, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11 + pxor m7, m7 + pshuflw m6, m6, q0000 + punpcklwd m6, m7 + pcmpgtb m4, m7, m0 + pcmpgtb m5, m7, m1 + punpcklbw m0, m4 + punpcklbw m1, m5 + DEFINE_ARGS buf, fg_data, h, x + pshufd m4, m1, q0000 + pshufd m5, m1, q1111 + pshufd m3, m0, q3333 + pshufd m2, m0, q2222 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + SCRATCH 0, 8, 0 + SCRATCH 1, 9, 1 + SCRATCH 2, 10, 2 + SCRATCH 3, 11, 3 + SCRATCH 4, 12, 4 + SCRATCH 5, 13, 5 + SCRATCH 6, 14, 6 + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar2: + mov xq, -76 + +.x_loop_ar2: + movq m0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + movhps m0, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pcmpgtb m2, m7, m0 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 + psrldq m5, m0, 2 ; y=-2,x=[-1,+5] + psrldq m3, m1, 2 ; y=-1,x=[-1,+5] + psrldq m4, m1, 4 ; y=-1,x=[+0,+5] + punpcklwd m2, m0, m5 + punpcklwd m3, m4 + pmaddwd m2, m8 + pmaddwd m3, m11 + paddd m2, m3 + + psrldq m4, m0, 4 ; y=-2,x=[+0,+5] + psrldq m5, m0, 6 ; y=-2,x=[+1,+5] + psrldq m6, m0, 8 ; y=-2,x=[+2,+5] + punpcklwd m4, m5 + punpcklwd m6, m1 + psrldq m5, m1, 6 ; y=-1,x=[+1,+5] + psrldq m1, m1, 8 ; y=-1,x=[+2,+5] + punpcklwd m5, m1 + pmaddwd m4, m9 + pmaddwd m6, m10 + pmaddwd m5, m12 + paddd m4, m6 + paddd m2, m5 + paddd m2, m4 + paddd m2, m14 + + movq m0, [bufq+xq-2] ; y=0,x=[-2,+5] +.x_loop_ar2_inner: + pcmpgtb m4, m7, m0 + punpcklbw m1, m0, m4 + pmaddwd m3, m1, m13 + paddd m3, m2 + psrldq m1, 4 ; y=0,x=0 + psrldq m2, 4 ; shift top to next pixel + psrad m3, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + paddw m3, m1 + packsswb m3, m3 + pslldq m3, 2 + pand m3, m15 + pandn m1, m15, m0 + por m0, m1, m3 + psrldq m0, 1 + ; overwrite 2 pixels, but that's ok + movd [bufq+xq-1], m0 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82 + dec hd + jg .y_loop_ar2 + RET + +.ar3: + DEFINE_ARGS buf, fg_data, shift +%if ARCH_X86_32 +%assign stack_offset stack_offset_old + ALLOC_STACK -16*14 +%elif WIN64 + SUB rsp, 16*6 +%assign stack_size_padded (stack_size_padded+16*6) +%assign stack_size (stack_size+16*6) +%else + ALLOC_STACK -16*6 +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m6, [base+round_vals-12+shiftq*2] + movd m7, [base+byte_blend] + movu m0, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15 + movq m2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 + pxor m3, m3 + pcmpgtb m4, m3, m0 + pcmpgtb m3, m2 + pshuflw m6, m6, q0000 + SCRATCH 6, 14, 12 + SCRATCH 7, 15, 13 + punpckhbw m1, m0, m4 + punpcklbw m0, m4 + punpcklbw m2, m3 + pshufd m3, m0, q1111 + pshufd m4, m0, q2222 + pshufd m5, m0, q3333 + pshufd m0, m0, q0000 + mova [rsp+ 0*16], m0 + mova [rsp+ 1*16], m3 + mova [rsp+ 2*16], m4 + mova [rsp+ 3*16], m5 + pshufd m6, m1, q1111 + pshufd m7, m1, q2222 + pshufd m5, m1, q3333 + pshufd m1, m1, q0000 + pshufd m3, m2, q1111 + psrldq m0, m2, 10 + pinsrw m2, [base+pw_1], 5 + pshufd m4, m2, q2222 + pshufd m2, m2, q0000 + pinsrw m0, [base+round_vals+shiftq*2-10], 3 + mova [rsp+ 4*16], m1 + mova [rsp+ 5*16], m6 + SCRATCH 7, 8, 6 + SCRATCH 5, 9, 7 + SCRATCH 2, 10, 8 + SCRATCH 3, 11, 9 + SCRATCH 4, 12, 10 + SCRATCH 0, 13, 11 + DEFINE_ARGS buf, fg_data, h, x + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar3: + mov xq, -76 + +.x_loop_ar3: + movu m0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] + pxor m3, m3 + pcmpgtb m3, m0 + punpckhbw m2, m0, m3 + punpcklbw m0, m3 + + psrldq m5, m0, 2 + psrldq m6, m0, 4 + psrldq m7, m0, 6 + punpcklwd m4, m0, m5 + punpcklwd m6, m7 + pmaddwd m4, [rsp+ 0*16] + pmaddwd m6, [rsp+ 1*16] + paddd m4, m6 + + movu m1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] + pxor m5, m5 + pcmpgtb m5, m1 + punpckhbw m3, m1, m5 + punpcklbw m1, m5 + palignr m6, m2, m0, 10 + palignr m7, m2, m0, 12 + psrldq m0, 8 + punpcklwd m0, m6 + punpcklwd m7, m1 + pmaddwd m0, [rsp+ 2*16] + pmaddwd m7, [rsp+ 3*16] + paddd m0, m7 + paddd m0, m4 + + psrldq m4, m1, 2 + psrldq m5, m1, 4 + psrldq m6, m1, 6 + psrldq m7, m1, 8 + punpcklwd m4, m5 + punpcklwd m6, m7 + pmaddwd m4, [rsp+ 4*16] + pmaddwd m6, [rsp+ 5*16] + paddd m4, m6 + paddd m0, m4 + + movu m2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + pxor m7, m7 + pcmpgtb m7, m2 + punpckhbw m5, m2, m7 + punpcklbw m2, m7 + palignr m7, m3, m1, 10 + palignr m3, m1, 12 + psrldq m1, m2, 2 + punpcklwd m7, m3 + punpcklwd m3, m2, m1 + pmaddwd m7, m8 + pmaddwd m3, m9 + paddd m7, m3 + paddd m0, m7 + + psrldq m6, m2, 4 + psrldq m1, m2, 6 + psrldq m3, m2, 8 + palignr m4, m5, m2, 10 + palignr m5, m5, m2, 12 + + punpcklwd m6, m1 + punpcklwd m3, m4 + punpcklwd m5, m14 + pmaddwd m6, m10 + pmaddwd m3, m11 + pmaddwd m5, m12 + paddd m0, m6 + paddd m3, m5 + paddd m0, m3 + + movq m1, [bufq+xq-3] ; y=0,x=[-3,+4] +.x_loop_ar3_inner: + pxor m5, m5 + pcmpgtb m5, m1 + punpcklbw m2, m1, m5 + pmaddwd m2, m13 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + packsswb m2, m2 + pslldq m2, 3 + pand m2, m15 + pandn m3, m15, m1 + por m1, m2, m3 + movd [bufq+xq-3], m1 + psrldq m1, 1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82 + dec hd + jg .y_loop_ar3 + RET + +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y +INIT_XMM ssse3 +cglobal generate_grain_uv_%1, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv + movifnidn r2, r2mp + movifnidn r3, r3mp + LEA r4, $$ +%define base r4-$$ + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r5d, [fg_dataq+FGData.grain_scale_shift] + movd m6, [base+round+r5*2] + mova m5, [base+pb_mask] + movd m0, [fg_dataq+FGData.seed] + movd m2, [base+pw_seed_xor+uvq*4] + pxor m0, m2 + pshuflw m6, m6, q0000 + pshuflw m0, m0, q0000 + lea r6, [base+gaussian_sequence] +%if %2 +%if ARCH_X86_64 + mov r7d, 73-35*%3 +%else + mov r3mp, 73-35*%3 +%endif + add bufq, 44 +.loop_y: + mov r5, -44 +.loop_x: +%else + mov r5, -82*73 + sub bufq, r5 +.loop: +%endif + pand m2, m0, m1 + psrlw m3, m2, 10 + por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m2, m4 ; bits 0x0f00 are set + pshufb m3, m5, m2 ; set 15th bit for next 4 seeds + psllq m2, m3, 30 + por m3, m2 + psllq m2, m3, 15 + por m3, m2 ; aggregate each bit into next seed's high bit + pmulhuw m2, m0, m7 + por m2, m3 ; 4 next output seeds + pshuflw m0, m2, q3333 + psrlw m2, 5 +%if ARCH_X86_64 + movd r9d, m2 + pshuflw m2, m2, q3232 + movzx r8, r9w + shr r9, 16 + + movd m3, [r6+r8*2] + pinsrw m3, [r6+r9*2], 1 + + movd r9d, m2 + movzx r8, r9w + shr r9, 16 + + pinsrw m3, [r6+r8*2], 2 + pinsrw m3, [r6+r9*2], 3 +%else + movd r2, m2 + pshuflw m2, m2, q3232 + movzx r1, r2w + shr r2, 16 + + movd m3, [r6+r1*2] + pinsrw m3, [r6+r2*2], 1 + + movd r2, m2 + movzx r1, r2w + shr r2, 16 + + pinsrw m3, [r6+r1*2], 2 + pinsrw m3, [r6+r2*2], 3 +%endif + pmulhrsw m3, m6 + packsswb m3, m3 + movd [bufq+r5], m3 + add r5, 4 +%if %2 + jl .loop_x + add bufq, 82 +%if ARCH_X86_64 + dec r7d +%else + dec r3mp +%endif + jg .loop_y +%else + jl .loop +%endif + +%if ARCH_X86_32 + mov r2, r2mp +%endif + + ; auto-regression code + movsxd r5, [fg_dataq+FGData.ar_coeff_lag] + movsxd r5, [base+generate_grain_uv_%1_ssse3_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_ssse3_table] + jmp r5 + +.ar0: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + movifnidn bufyq, bufymp +%if ARCH_X86_32 +%assign stack_offset_old stack_offset + ALLOC_STACK -2*16 +%endif + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m5, [fg_dataq+FGData.ar_coeffs_uv+uvq] + movd m4, [base+hmul_bits+shiftq*2] + DEFINE_ARGS buf, bufy, h, x + pxor m0, m0 + pcmpgtb m0, m5 + punpcklbw m5, m0 + movd m7, [base+pb_1] +%if %2 + movd m6, [base+hmul_bits+2+%3*2] +%endif + pshuflw m5, m5, q0000 + pshuflw m4, m4, q0000 + pshufd m7, m7, q0000 +%if %2 + pshuflw m6, m6, q0000 +%endif + punpcklqdq m5, m5 + punpcklqdq m4, m4 +%if %2 + punpcklqdq m6, m6 +%endif + pcmpeqw m1, m1 + pslldq m1, 12>>%2 + SCRATCH 1, 8, 0 + SCRATCH 4, 9, 1 +%if %2 + sub bufq, 82*(73-35*%3)+82-(82*3+41) +%else + sub bufq, 82*70-3 +%endif + add bufyq, 3+82*3 + mov hd, 70-35*%3 +.y_loop_ar0: + xor xd, xd +.x_loop_ar0: + ; first 32 pixels +%if %2 + movu m1, [bufyq+xq*2] +%if %3 + movu m2, [bufyq+xq*2+82] +%endif + movu m3, [bufyq+xq*2+16] +%if %3 + movu m4, [bufyq+xq*2+82+16] +%endif + pmaddubsw m0, m7, m1 +%if %3 + pmaddubsw m1, m7, m2 +%endif + pmaddubsw m2, m7, m3 +%if %3 + pmaddubsw m3, m7, m4 + paddw m0, m1 + paddw m2, m3 +%endif + pmulhrsw m0, m6 + pmulhrsw m2, m6 +%else + movu m0, [bufyq+xq] + pxor m6, m6 + pcmpgtb m6, m0 + punpckhbw m2, m0, m6 + punpcklbw m0, m6 +%endif + pmullw m0, m5 + pmullw m2, m5 + pmulhrsw m0, m9 + pmulhrsw m2, m9 + movu m1, [bufq+xq] + pxor m4, m4 + pcmpgtb m4, m1 + punpckhbw m3, m1, m4 +%if %2 + punpcklbw m1, m4 + paddw m2, m3 + paddw m0, m1 +%else + punpcklbw m6, m1, m4 + paddw m2, m3 + paddw m0, m6 +%endif + packsswb m0, m2 +%if %2 + movu [bufq+xq], m0 + add xd, 16 + cmp xd, 32 + jl .x_loop_ar0 + + ; last 6/12 pixels + movu m1, [bufyq+xq*(1+%2)] +%if %3 + movu m2, [bufyq+xq*2+82] +%endif + pmaddubsw m0, m7, m1 +%if %3 + pmaddubsw m1, m7, m2 + paddw m0, m1 +%endif + pmulhrsw m0, m6 + pmullw m0, m5 + pmulhrsw m0, m9 + movq m1, [bufq+xq] + pxor m4, m4 + pcmpgtb m4, m1 + punpcklbw m2, m1, m4 + paddw m0, m2 + packsswb m0, m0 + pandn m2, m8, m0 + pand m1, m8 + por m2, m1 + movq [bufq+xq], m2 +%else + add xd, 16 + cmp xd, 80 + je .y_loop_final_ar0 + movu [bufq+xq-16], m0 + jmp .x_loop_ar0 +.y_loop_final_ar0: + pandn m2, m8, m0 + pand m1, m8 + por m2, m1 + movu [bufq+xq-16], m2 +%endif + + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar0 + RET + +.ar1: +%if ARCH_X86_32 +%assign stack_offset stack_offset_old +%assign stack_size_padded 0 +%xdefine rstk rsp +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x + imul uvd, 28 + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] + movd m4, [fg_dataq+FGData.ar_coeffs_uv+uvq-1] + pinsrw m4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 2 +%if ARCH_X86_32 + mov r3mp, cf3d + DEFINE_ARGS buf, shift, fg_data, val3, min, max, x +%elif WIN64 + DEFINE_ARGS shift, bufy, fg_data, buf, val3, cf3, min, max, x + mov bufq, r0 +%else + DEFINE_ARGS buf, bufy, fg_data, shift, val3, cf3, min, max, x +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m3, [base+round_vals+shiftq*2-12] ; rnd +%if %2 + movd m7, [base+pb_1] + movd m6, [base+hmul_bits+2+%3*2] +%endif + psrldq m4, 1 +%if ARCH_X86_32 + DEFINE_ARGS buf, shift, val0, val3, min, max, x +%elif WIN64 + DEFINE_ARGS shift, bufy, h, buf, val3, cf3, min, max, x, val0 +%else + DEFINE_ARGS buf, bufy, h, shift, val3, cf3, min, max, x, val0 +%endif + pxor m5, m5 + punpcklwd m3, m5 +%if %2 + punpcklwd m6, m6 +%endif + pcmpgtb m5, m4 + punpcklbw m4, m5 + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + pshufd m3, m3, q0000 +%if %2 + pshufd m7, m7, q0000 + pshufd m6, m6, q0000 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif +%if ARCH_X86_32 + add r1mp, 79+82*3 + mov r0mp, 70-35*%3 +%else + add bufyq, 79+82*3 + mov hd, 70-35*%3 +%endif + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -(76>>%2) + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: +%if %2 +%if ARCH_X86_32 + mov r2, r1mp + movq m0, [r2+xq*2] +%if %3 + movq m1, [r2+xq*2+82] +%endif +%else + movq m0, [bufyq+xq*2] +%if %3 + movq m1, [bufyq+xq*2+82] +%endif +%endif + pmaddubsw m2, m7, m0 +%if %3 + pmaddubsw m0, m7, m1 + paddw m2, m0 +%endif + pmulhrsw m2, m6 +%else +%if ARCH_X86_32 + mov r2, r1mp + movd m2, [r2+xq] +%else + movd m2, [bufyq+xq] +%endif + pxor m0, m0 + pcmpgtb m0, m2 + punpcklbw m2, m0 +%endif + + movq m0, [bufq+xq-82-1] ; top/left + pxor m1, m1 + pcmpgtb m1, m0 + punpcklbw m0, m1 + psrldq m1, m0, 4 ; top/right + punpcklwd m1, m2 + psrldq m2, m0, 2 ; top + punpcklwd m0, m2 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 + paddd m0, m3 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 +%if ARCH_X86_32 + imul val3d, r3mp +%else + imul val3d, cf3d +%endif + add val3d, val0d + sar val3d, shiftb + movsx val0d, byte [bufq+xq] + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov byte [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82 +%if ARCH_X86_32 + add r1mp, 82<<%3 + dec r0mp +%else + add bufyq, 82<<%3 + dec hd +%endif + jg .y_loop_ar1 + RET + +.ar2: +%if ARCH_X86_32 +%assign stack_offset stack_offset_old +%assign stack_size_padded 0 +%xdefine rstk rsp + ALLOC_STACK -8*16 +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + movifnidn bufyq, bufymp + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + movd m7, [base+round_vals-12+shiftq*2] + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-12 + pxor m2, m2 + pcmpgtb m2, m0 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 + pinsrw m1, [base+pw_1], 5 + punpcklwd m7, m7 + pshufd m7, m7, q0000 + DEFINE_ARGS buf, bufy, fg_data, h, unused, x + pshufd m4, m1, q0000 + pshufd m5, m1, q1111 + pshufd m6, m1, q2222 + pshufd m3, m0, q3333 + pshufd m2, m0, q2222 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + SCRATCH 0, 8, 0 + SCRATCH 1, 9, 1 + SCRATCH 2, 10, 2 + SCRATCH 3, 11, 3 + SCRATCH 4, 12, 4 + SCRATCH 5, 13, 5 + SCRATCH 6, 14, 6 + SCRATCH 7, 15, 7 +%if %2 + movd m7, [base+hmul_bits+2+%3*2] + movd m6, [base+pb_1] + punpcklwd m7, m7 + pshufd m6, m6, q0000 + pshufd m7, m7, q0000 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar2: + mov xq, -(76>>%2) + +.x_loop_ar2: + pxor m2, m2 + movq m0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + movhps m0, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pcmpgtb m2, m0 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 + psrldq m5, m0, 2 ; y=-2,x=[-1,+5] + psrldq m3, m1, 2 ; y=-1,x=[-1,+5] + psrldq m4, m1, 4 ; y=-1,x=[+0,+5] + punpcklwd m2, m0, m5 + punpcklwd m3, m4 + pmaddwd m2, m8 + pmaddwd m3, m11 + paddd m2, m3 + + psrldq m4, m0, 4 ; y=-2,x=[+0,+5] + psrldq m5, m0, 6 ; y=-2,x=[+1,+5] + psrldq m0, 8 ; y=-2,x=[+2,+5] + punpcklwd m4, m5 + punpcklwd m0, m1 + psrldq m3, m1, 6 ; y=-1,x=[+1,+5] + psrldq m1, m1, 8 ; y=-1,x=[+2,+5] + punpcklwd m3, m1 + pmaddwd m4, m9 + pmaddwd m0, m10 + pmaddwd m3, m12 + paddd m4, m0 + paddd m2, m3 + paddd m2, m4 + +%if %2 + movq m1, [bufyq+xq*2] +%if %3 + movq m3, [bufyq+xq*2+82] +%endif + pmaddubsw m0, m6, m1 +%if %3 + pmaddubsw m1, m6, m3 + paddw m0, m1 +%endif + pmulhrsw m0, m7 +%else + movd m0, [bufyq+xq] + pxor m1, m1 + pcmpgtb m1, m0 + punpcklbw m0, m1 +%endif + punpcklwd m0, m15 + pmaddwd m0, m14 + paddd m2, m0 + + movq m0, [bufq+xq-2] ; y=0,x=[-2,+5] + pxor m4, m4 + movd m5, [base+byte_blend+1] + punpcklbw m5, m5 +.x_loop_ar2_inner: + pcmpgtb m1, m4, m0 + punpcklbw m0, m1 + pmaddwd m3, m0, m13 + paddd m3, m2 + psrldq m2, 4 ; shift top to next pixel + psrad m3, [fg_dataq+FGData.ar_coeff_shift] + pslldq m3, 4 + pand m3, m5 + paddw m0, m3 + packsswb m0, m0 + movd [bufq+xq-2], m0 + psrldq m0, 1 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar2 + RET + +.ar3: +%if ARCH_X86_32 +%assign stack_offset stack_offset_old +%assign stack_size_padded 0 +%xdefine rstk rsp +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + movifnidn bufyq, bufymp +%if ARCH_X86_32 + ALLOC_STACK -15*16 +%else + SUB rsp, 16*7 +%assign stack_size_padded (stack_size_padded+16*7) +%assign stack_size (stack_size+16*7) +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-15 + pxor m3, m3 + pcmpgtb m3, m0 + punpckhbw m1, m0, m3 + punpcklbw m0, m3 + pshufd m2, m0, q1111 + pshufd m3, m0, q2222 + pshufd m4, m0, q3333 + pshufd m0, m0, q0000 + pshufd m5, m1, q1111 + pshufd m6, m1, q2222 + pshufd m7, m1, q3333 + pshufd m1, m1, q0000 + mova [rsp+ 0*16], m0 + mova [rsp+ 1*16], m2 + mova [rsp+ 2*16], m3 + mova [rsp+ 3*16], m4 + mova [rsp+ 4*16], m1 + mova [rsp+ 5*16], m5 + mova [rsp+ 6*16], m6 + SCRATCH 7, 8, 7 + + movu m2, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-24 [24=luma] + pxor m4, m4 + pcmpgtb m4, m2 + punpckhbw m5, m2, m4 + punpcklbw m2, m4 + pshufd m4, m2, q3232 + punpcklwd m3, m4, m5 + pshuflw m5, m4, q3321 + pshufd m4, m3, q0000 + pshufd m3, m2, q1111 + pshufd m2, m2, q0000 + pinsrw m5, [base+round_vals+shiftq*2-10], 3 + SCRATCH 2, 9, 8 + SCRATCH 3, 10, 9 + SCRATCH 4, 11, 10 + SCRATCH 5, 12, 11 + + movd m2, [base+round_vals-12+shiftq*2] +%if %2 + movd m1, [base+pb_1] + movd m3, [base+hmul_bits+2+%3*2] +%endif + pxor m0, m0 + punpcklwd m2, m0 +%if %2 + punpcklwd m3, m3 +%endif + pshufd m2, m2, q0000 +%if %2 + pshufd m1, m1, q0000 + pshufd m3, m3, q0000 + SCRATCH 1, 13, 12 +%endif + SCRATCH 2, 14, 13 +%if %2 + SCRATCH 3, 15, 14 +%endif + + DEFINE_ARGS buf, bufy, fg_data, h, unused, x +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar3: + mov xq, -(76>>%2) + +.x_loop_ar3: + movu m0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] + pxor m4, m4 + pcmpgtb m4, m0 + punpckhbw m3, m0, m4 + punpcklbw m0, m4 + + psrldq m5, m0, 2 + psrldq m6, m0, 4 + psrldq m7, m0, 6 + punpcklwd m4, m0, m5 + punpcklwd m6, m7 + pmaddwd m4, [rsp+ 0*16] + pmaddwd m6, [rsp+ 1*16] + paddd m4, m6 + + palignr m2, m3, m0, 10 + palignr m3, m0, 12 + psrldq m0, 8 + + movu m1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] + pxor m6, m6 + pcmpgtb m6, m1 + punpckhbw m5, m1, m6 + punpcklbw m1, m6 + + punpcklwd m0, m2 + punpcklwd m3, m1 + pmaddwd m0, [rsp+ 2*16] + pmaddwd m3, [rsp+ 3*16] + paddd m0, m3 + paddd m0, m4 + + movu m2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + pxor m7, m7 + pcmpgtb m7, m2 + punpckhbw m6, m2, m7 + punpcklbw m2, m7 + + palignr m3, m5, m1, 10 + palignr m5, m1, 12 + psrldq m4, m2, 2 + + punpcklwd m3, m5 + punpcklwd m5, m2, m4 + pmaddwd m3, [rsp+ 6*16] + pmaddwd m5, m8 + paddd m3, m5 + paddd m0, m3 + + psrldq m3, m1, 2 + psrldq m4, m1, 4 + psrldq m5, m1, 6 + psrldq m1, 8 + + punpcklwd m3, m4 + punpcklwd m5, m1 + pmaddwd m3, [rsp+ 4*16] + pmaddwd m5, [rsp+ 5*16] + paddd m3, m5 + paddd m0, m3 + +%if %2 + movq m1, [bufyq+xq*2] +%if %3 + movq m3, [bufyq+xq*2+82] +%endif + pmaddubsw m7, m13, m1 +%if %3 + pmaddubsw m5, m13, m3 + paddw m7, m5 +%endif + pmulhrsw m7, m15 +%else + movd m7, [bufyq+xq] + pxor m1, m1 + pcmpgtb m1, m7 + punpcklbw m7, m1 +%endif + + psrldq m1, m2, 4 + psrldq m3, m2, 6 + palignr m4, m6, m2, 10 + palignr m6, m2, 12 + psrldq m2, 8 + + punpcklwd m1, m3 + punpcklwd m2, m4 + punpcklwd m6, m7 + pmaddwd m1, m9 + pmaddwd m2, m10 + pmaddwd m6, m11 + paddd m1, m2 + paddd m0, m6 + paddd m0, m1 + paddd m0, m14 + + movq m1, [bufq+xq-3] ; y=0,x=[-3,+4] + pxor m4, m4 + movd m5, [base+byte_blend] +.x_loop_ar3_inner: + pcmpgtb m2, m4, m1 + punpcklbw m3, m1, m2 + pmaddwd m2, m3, m12 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw, we only care about one value + packsswb m2, m2 + pandn m3, m5, m1 + pslld m2, 24 + pand m2, m5 + por m1, m2, m3 + movd [bufq+xq-3], m1 + psrldq m1, 1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar3 + RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 + +%macro vpgatherdw 5-6 ; dst, src, base, tmp_gpr[x2], tmp_xmm_reg +%assign %%idx 0 +%define %%tmp %2 +%if %0 == 6 +%define %%tmp %6 +%endif +%rep 4 +%if %%idx == 0 + movd %5 %+ d, %2 + pshuflw %%tmp, %2, q3232 +%else + movd %5 %+ d, %%tmp +%if %%idx == 2 + punpckhqdq %%tmp, %%tmp +%elif %%idx == 4 + psrlq %%tmp, 32 +%endif +%endif + movzx %4 %+ d, %5 %+ w + shr %5 %+ d, 16 + +%if %%idx == 0 + movd %1, [%3+%4] +%else + pinsrw %1, [%3+%4], %%idx + 0 +%endif + pinsrw %1, [%3+%5], %%idx + 1 +%assign %%idx %%idx+2 +%endrep +%endmacro + +INIT_XMM ssse3 +; fgy_32x32xn(dst, src, stride, fg_data, w, scaling, grain_lut, h, sby) +%if ARCH_X86_32 +%if STACK_ALIGNMENT < mmsize +cglobal fgy_32x32xn, 0, 7, 16, 0 - (6 * mmsize + (9 + 3) * gprsize), \ + dst, src, scaling, unused1, fg_data, picptr, unused2 + ; copy stack arguments to new position post-alignment, so that we + ; don't have to keep the old stack location in a separate register + mov r0, r0m + mov r1, r2m + mov r2, r4m + mov r3, r6m + mov r4, r7m + mov r5, r8m + + mov [rsp+6*mmsize+ 3*gprsize], r0 + mov [rsp+6*mmsize+ 5*gprsize], r1 + mov [rsp+6*mmsize+ 7*gprsize], r2 + mov [rsp+6*mmsize+ 9*gprsize], r3 + mov [rsp+6*mmsize+10*gprsize], r4 + mov [rsp+6*mmsize+11*gprsize], r5 +%else +cglobal fgy_32x32xn, 0, 7, 16, 6 * mmsize + (3 + 1) * gprsize, \ + dst, src, scaling, unused1, fg_data, picptr, unused2 +%endif + mov srcq, srcm + mov fg_dataq, r3m + mov scalingq, r5m +%if STACK_ALIGNMENT < mmsize +%define r0m [rsp+6*mmsize+ 3*gprsize] +%define r1m [rsp+6*mmsize+ 4*gprsize] +%define r2m [rsp+6*mmsize+ 5*gprsize] +%define r3m [rsp+6*mmsize+ 6*gprsize] +%define r4m [rsp+6*mmsize+ 7*gprsize] +%define r5m [rsp+6*mmsize+ 8*gprsize] +%define r6m [rsp+6*mmsize+ 9*gprsize] +%define r7m [rsp+6*mmsize+10*gprsize] +%define r8m [rsp+6*mmsize+11*gprsize] +%endif + LEA r5, pb_mask +%define base r5-pb_mask + mov r5m, picptrq +%else +cglobal fgy_32x32xn, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut + lea r7, [pb_mask] +%define base r7-pb_mask +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + movd m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + pcmpeqw m2, m2 + psrldq m2, 14 + movd m4, [base+max+r6*4] + movd m5, [base+min+r6*2] + punpcklwd m3, m3 + punpcklwd m4, m4 + punpcklwd m5, m5 + pshufd m3, m3, q0000 + pshufd m4, m4, q0000 + pshufd m5, m5, q0000 + SCRATCH 2, 10, 0 + SCRATCH 3, 11, 1 + SCRATCH 4, 12, 2 + SCRATCH 5, 13, 3 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap +%endif + + mov sbyd, r8m + mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1 + test overlapd, overlapd + jz .no_vertical_overlap + mova m6, [base+pw_1024] + movd m7, [base+pb_27_17_17_27] + SCRATCH 6, 14, 4 + SCRATCH 7, 15, 5 + test sbyd, sbyd + jnz .vertical_overlap + ; fall-through + +.no_vertical_overlap: + mov r8m, overlapd +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, unused + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + unused1, unused2, see, unused3 +%endif + + lea src_bakq, [srcq+wq] + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r4m, wq + DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3 +%endif + +.loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, unused + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr, + ; r6m=grain_lut, r7m=h, r8m=overlap_v|h + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, unused +%endif + +.loop_x_odd: + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y: + ; src + mova m0, [srcq] + pxor m2, m2 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq, r0, r5, m3 + vpgatherdw m5, m1, scalingq, r0, r5, m3 +%else + vpgatherdw m4, m0, scalingq, r12, r13, m3 + vpgatherdw m5, m1, scalingq, r12, r13, m3 +%endif + pcmpeqw m3, m3 + psrlw m3, 8 + pand m4, m3 + pand m5, m3 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] + pcmpgtb m7, m2, m3 + punpcklbw m2, m3, m7 + punpckhbw m3, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m2, m4 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + + add srcq, r2mp + add grain_lutq, 82 + dec hd + jg .loop_y + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r1mp + add srcq, r4mp +%else + lea srcq, [src_bakq+wq] +%endif + btc dword r8m, 2 + jc .next_blk + + add offxyd, 16 + test dword r8m, 2 ; r8m & 2 = have_top_overlap + jz .loop_x_odd + +%if ARCH_X86_32 + add dword [rsp+6*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxyd +%endif + jnz .loop_x_odd_v_overlap + +.next_blk: + test dword r8m, 1 + jz .loop_x + + test dword r8m, 2 + jnz .loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: +%if ARCH_X86_32 + ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr, + ; r6m=grain_lut, r7m=h, r8m=overlap_v|h + DEFINE_ARGS dst, src, scaling, offxy, unused1, unused2, unused3 + + add offxyd, 16 ; left_offxyd + mov [rsp+6*mmsize+0*gprsize], offxyd + + DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3 + + mov seed, r3m +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy + + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx +%endif + + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y_h_overlap: + ; src + mova m0, [srcq] + pxor m2, m2 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq, r0, r5, m3 + vpgatherdw m5, m1, scalingq, r0, r5, m3 +%else + vpgatherdw m4, m0, scalingq, r12, r13, m3 + vpgatherdw m5, m1, scalingq, r12, r13, m3 +%endif + pcmpeqw m3, m3 + psrlw m3, 8 + pand m4, m3 + pand m5, m3 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r5, [rsp+6*mmsize+0*gprsize] + movd m7, [grain_lutq+r5] +%else + movd m7, [grain_lutq+left_offxyq] +%endif + punpcklbw m7, m3 + pmaddubsw m6, m15, m7 + pmulhrsw m6, m14 + packsswb m6, m6 + pand m6, m10 + pandn m7, m10, m3 + por m6, m7 + pcmpgtb m2, m6 + punpcklbw m7, m6, m2 + punpckhbw m6, m2 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m7, m4 + pmullw m6, m5 + pmulhrsw m7, m11 + pmulhrsw m6, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m7 + paddw m1, m6 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + + add srcq, r2mp + add grain_lutq, 82 + dec hd + jg .loop_y_h_overlap + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r1m + add srcq, r4m +%else + lea srcq, [src_bakq+wq] +%endif + xor dword r8m, 4 + add offxyd, 16 + + ; since this half-block had left-overlap, the next does not + test dword r8m, 2 ; have_top_overlap + jz .loop_x_odd +%if ARCH_X86_32 + add dword [rsp+6*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxyd +%endif + jmp .loop_x_odd_v_overlap + +.end: + RET + +.vertical_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap +%endif + + or overlapd, 2 ; top_overlap: overlap & 2 + mov r8m, overlapd + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul tmpd, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add tmpd, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and tmpd, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, tmpd +%if ARCH_X86_32 + xor sbyd, seed ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + tmp, unused2, see, unused3 +%endif + + lea src_bakq, [srcq+wq] + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r4m, wq + DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 +%endif + +.loop_x_v_overlap: +%if ARCH_X86_32 + mov seed, r3m +%endif + ; we assume from the block above that bits 8-15 of tmpd are zero'ed, + ; because of the 'and tmpd, 0x00ff00ff' above + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, unused, top_offxy + + mov offyd, seed + mov offxd, seed +%endif + + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, unused, top_offxy +%endif + + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+6*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +.loop_x_odd_v_overlap: +%if ARCH_X86_32 + mov r5, r5m + lea r5, [base+pb_27_17] + mov [rsp+5*mmsize+8], r5 +%else + mova m8, [pb_27_17] +%endif + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y_v_overlap: + ; src + mova m0, [srcq] + pxor m2, m2 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq, r0, r5, m3 + vpgatherdw m5, m1, scalingq, r0, r5, m3 +%else + vpgatherdw m4, m0, scalingq, r12, r13, m3 + vpgatherdw m5, m1, scalingq, r12, r13, m3 +%endif + pcmpeqw m3, m3 + psrlw m3, 8 + pand m4, m3 + pand m5, m3 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r5, [rsp+6*mmsize+1*gprsize] + movu m7, [grain_lutq+r5] +%else + movu m7, [grain_lutq+top_offxyq] +%endif + punpckhbw m6, m7, m3 + punpcklbw m7, m3 +%if ARCH_X86_32 + mov r5, [rsp+5*mmsize+8] + pmaddubsw m3, [r5], m6 + pmaddubsw m6, [r5], m7 +%else + pmaddubsw m3, m8, m6 + pmaddubsw m6, m8, m7 +%endif + pmulhrsw m3, m14 + pmulhrsw m6, m14 + packsswb m6, m3 + pcmpgtb m7, m2, m6 + punpcklbw m2, m6, m7 + punpckhbw m6, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m2, m4 + pmullw m6, m5 + pmulhrsw m2, m11 + pmulhrsw m6, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m6 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add dword [rsp+5*mmsize+8], mmsize +%else + mova m8, [pb_17_27] +%endif + add srcq, r2mp + add grain_lutq, 82 + dec hw + jz .end_y_v_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + btc hd, 16 + jnc .loop_y_v_overlap + jmp .loop_y + +.end_y_v_overlap: +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov srcq, r1mp + add srcq, r4mp +%else + lea srcq, [src_bakq+wq] +%endif + btc dword r8m, 2 + jc .loop_x_hv_overlap + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+6*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + jmp .loop_x_odd_v_overlap + +.loop_x_hv_overlap: +%if ARCH_X86_32 + mov r5, r5m + lea r5, [base+pb_27_17] + mov [rsp+5*mmsize+8], r5 + + DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, src_bak + + mov r5, [rsp+6*mmsize+1*gprsize] + mov r4, offxyd + add r5, 16 + add r4, 16 + mov [rsp+6*mmsize+2*gprsize], r5 ; topleft_offxy + mov [rsp+6*mmsize+0*gprsize], r4 ; left_offxy + + DEFINE_ARGS tmp, src, scaling, see, w, picptr, src_bak + + xor tmpd, tmpd + mov seed, r3m +%else + mova m8, [pb_27_17] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + tmp, unused2, see, unused3 + + ; we assume from the block above that bits 8-15 of tmpd are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offyq+16] + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut + + movzx r5, offxyw ; top_offxy + mov [rsp+6*mmsize+1*gprsize], r5 +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy + + movzx top_offxyd, offxyw +%endif + shr offxyd, 16 + + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r5, [rsp+6*mmsize+1*gprsize] ; top_offxy + mov r0, [rsp+6*mmsize+0*gprsize] ; left_offxy + movu m6, [grain_lutq+r5] + mov r5, [rsp+6*mmsize+2*gprsize] ; topleft_offxy + movd m4, [grain_lutq+r0] + movd m7, [grain_lutq+r5] +%else + movu m6, [grain_lutq+top_offxyq] + movd m4, [grain_lutq+left_offxyq] + movd m7, [grain_lutq+topleft_offxyq] +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m4, m3 + punpcklbw m7, m6 + pmaddubsw m2, m15, m4 + pmaddubsw m4, m15, m7 + pmulhrsw m2, m14 + pmulhrsw m4, m14 + packsswb m2, m2 + packsswb m4, m4 + pand m2, m10 + pand m4, m10 + pandn m7, m10, m3 + pandn m3, m10, m6 + por m7, m2 + por m3, m4 + ; followed by v interpolation (top | cur -> cur) + punpckhbw m4, m3, m7 + punpcklbw m3, m7 +%if ARCH_X86_32 + mov r5, [rsp+5*mmsize+8] + pmaddubsw m7, [r5], m4 + pmaddubsw m4, [r5], m3 +%else + pmaddubsw m7, m8, m4 + pmaddubsw m4, m8, m3 +%endif + pmulhrsw m7, m14 + pmulhrsw m4, m14 + packsswb m4, m7 + pxor m2, m2 + pcmpgtb m7, m2, m4 + punpcklbw m3, m4, m7 + punpckhbw m4, m7 + + ; src + mova m0, [srcq] + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m5, m0, scalingq, r0, r5, m7 + vpgatherdw m6, m1, scalingq, r0, r5, m7 +%else + vpgatherdw m5, m0, scalingq, r13, r14, m7 + vpgatherdw m6, m1, scalingq, r13, r14, m7 +%endif + pcmpeqw m7, m7 + psrlw m7, 8 + pand m5, m7 + pand m6, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m3, m5 + pmullw m4, m6 + pmulhrsw m3, m11 + pmulhrsw m4, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m3 + paddw m1, m4 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add dword [rsp+5*mmsize+8], mmsize +%else + mova m8, [pb_17_27] +%endif + add srcq, r2mp + add grain_lutq, 82 + dec hw + jz .end_y_hv_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + btc hd, 16 + jnc .loop_y_hv_overlap + jmp .loop_y_h_overlap + +.end_y_hv_overlap: +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov srcq, r1m + add srcq, r4m +%else + lea srcq, [src_bakq+wq] +%endif + xor dword r8m, 4 + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+6*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + jmp .loop_x_odd_v_overlap + +.end_hv: + RET + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +INIT_XMM ssse3 +%if ARCH_X86_32 +; fguv_32x32xn_i420_ssse3(dst, src, stride, fg_data, w, scaling, grain_lut, h, +; sby, luma, lstride, uv_pl, is_id) +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8 +cglobal fguv_32x32xn_i%1, 0, 7, 8, 0 - (8 * mmsize + (13 + 3) * gprsize), \ + tmp, src, scaling, h, fg_data, picptr, unused + mov r0, r0m + mov r1, r2m + mov r2, r4m + mov r3, r6m + mov r4, r7m + mov [rsp+8*mmsize+3*gprsize], r0 + mov [rsp+8*mmsize+5*gprsize], r1 + mov [rsp+8*mmsize+7*gprsize], r2 + mov [rsp+8*mmsize+9*gprsize], r3 + mov [rsp+8*mmsize+10*gprsize], r4 + + mov r0, r8m + mov r1, r9m + mov r2, r10m + mov r4, r11m + mov r3, r12m + mov [rsp+8*mmsize+11*gprsize], r0 + mov [rsp+8*mmsize+12*gprsize], r1 + mov [rsp+8*mmsize+13*gprsize], r2 + mov [rsp+8*mmsize+14*gprsize], r4 +%else +cglobal fguv_32x32xn_i%1, 0, 7, 8, 8 * mmsize + (4) * gprsize, \ + tmp, src, scaling, h, fg_data, picptr, unused +%endif + mov srcq, srcm + mov fg_dataq, r3m + mov scalingq, r5m +%if STACK_ALIGNMENT < mmsize +%define r0m [rsp+8*mmsize+ 3*gprsize] +%define r1m [rsp+8*mmsize+ 4*gprsize] +%define r2m [rsp+8*mmsize+ 5*gprsize] +%define r3m [rsp+8*mmsize+ 6*gprsize] +%define r4m [rsp+8*mmsize+ 7*gprsize] +%define r5m [rsp+8*mmsize+ 8*gprsize] +%define r6m [rsp+8*mmsize+ 9*gprsize] +%define r7m [rsp+8*mmsize+10*gprsize] +%define r8m [rsp+8*mmsize+11*gprsize] +%define r9m [rsp+8*mmsize+12*gprsize] +%define r10m [rsp+8*mmsize+13*gprsize] +%define r11m [rsp+8*mmsize+14*gprsize] +%define r12m [rsp+8*mmsize+15*gprsize] +%endif + LEA r5, pb_mask +%define base r5-pb_mask + mov r5m, r5 +%else +cglobal fguv_32x32xn_i%1, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, tmp, sby, luma, lstride, uv_pl, is_id + lea r8, [pb_mask] +%define base r8-pb_mask +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + pcmpeqw m2, m2 + movd m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + lea tmpd, [r6d*2] +%if ARCH_X86_32 && STACK_ALIGNMENT < mmsize + test r3, r3 +%else + cmp dword r12m, 0 ; is_idm +%endif + movd m5, [base+min+r6*2] + cmovne r6d, tmpd + movd m4, [base+max+r6*2] + psrldq m2, 14+%2 + punpcklwd m3, m3 + punpcklwd m5, m5 + punpcklwd m4, m4 + pshufd m3, m3, q0000 + pshufd m5, m5, q0000 + pshufd m4, m4, q0000 + SCRATCH 2, 10, 0 + SCRATCH 3, 11, 1 + SCRATCH 4, 12, 2 + SCRATCH 5, 13, 3 + + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap +%endif + +%if %1 + mov r6d, dword r11m + movd m0, [fg_dataq+FGData.uv_mult+r6*4] + movd m1, [fg_dataq+FGData.uv_luma_mult+r6*4] + punpcklbw m6, m1, m0 + movd m7, [fg_dataq+FGData.uv_offset+r6*4] + punpcklwd m6, m6 + punpcklwd m7, m7 + pshufd m6, m6, q0000 + pshufd m7, m7, q0000 + SCRATCH 6, 14, 4 + SCRATCH 7, 15, 5 +%endif + + mov sbyd, r8m + mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1 + test overlapd, overlapd + jz %%no_vertical_overlap +%if ARCH_X86_32 +%if %2 + movd m1, [base+pb_23_22] +%else + movd m1, [base+pb_27_17_17_27] +%endif + mova m0, [base+pw_1024] +%else +%if %2 + movd m1, [pb_23_22] +%else + movd m1, [pb_27_17_17_27] +%endif + mova m0, [pw_1024] +%endif + pshufd m1, m1, q0000 + SCRATCH 0, 8, 6 + SCRATCH 1, 9, 7 + test sbyd, sbyd + jnz %%vertical_overlap + ; fall-through + +%%no_vertical_overlap: + mov r8m, overlapd +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, overlap + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak +%define luma_bakq lumaq + + mov wq, r4m +%if %3 + shl r10mp, 1 +%endif +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + unused2, unused3, see, overlap, unused4, src_bak, lstride, luma_bak + + mov lstrideq, r10mp +%endif + + mov lumaq, r9mp + lea src_bakq, [srcq+wq] + lea luma_bakq, [lumaq+wq*(1+%2)] + neg wq + sub r0mp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r11m, luma_bakq + mov r4m, wq + + DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 +%else + mov r11mp, src_bakq + mov r12mp, strideq +%endif + +%%loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, unused1, unused2, lstride + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, unused1, unused2, lstride, luma_bak +%endif + +%%loop_x_odd: + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y: + ; src +%if ARCH_X86_32 + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq, r0, r5 + vpgatherdw m5, m6, scalingq, r0, r5 +%else + vpgatherdw m7, m4, scalingq, r12, r2 + vpgatherdw m5, m6, scalingq, r12, r2 +%endif + pcmpeqw m1, m1 + psrlw m1, 8 + pand m7, m1 + pand m5, m1 + + ; unpack chroma_source + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq+ 0] + pcmpgtb m6, m2, m3 + punpcklbw m2, m3, m6 + punpckhbw m3, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add srcq, r2mp + ; we already incremented lumaq above +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif +%endif + add grain_lutq, 82 + dec hw + jg %%loop_y + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif +%if %2 == 0 + ; adjust top_offxy +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + add offxyd, 16 + btc dword r8m, 2 + jc %%loop_x_even + test dword r8m, 2 + jz %%loop_x_odd + jmp %%loop_x_odd_v_overlap +%%loop_x_even: +%endif + test dword r8m, 1 + jz %%loop_x + + ; r8m = sbym + test dword r8m, 2 + jne %%loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: +%if ARCH_X86_32 +%if %2 + lea r6, [offxyd+16] + mov [rsp+8*mmsize+0*gprsize], r6 +%else + mov [rsp+8*mmsize+0*gprsize], offxyd +%endif + + DEFINE_ARGS luma, src, scaling, see, w, picptr, grain_lut + + mov seed, r3m +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, lstride + +%if %2 + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx +%else + mov left_offxyd, offyd +%endif +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS luma, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, lstride + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, unused1, unused2, lstride, luma_bak +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_h_overlap: + ; src +%if ARCH_X86_32 + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq, r0, r5 + vpgatherdw m5, m6, scalingq, r0, r5 +%else + vpgatherdw m7, m4, scalingq, r12, r2 + vpgatherdw m5, m6, scalingq, r12, r2 +%endif + pcmpeqw m1, m1 + psrlw m1, 8 + pand m7, m1 + pand m5, m1 + + ; unpack chroma_source + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq+ 0] +%if ARCH_X86_32 + mov r0, [rsp+8*mmsize+0*gprsize] + movd m4, [grain_lutq+r0+ 0] +%else + movd m4, [grain_lutq+left_offxyq+ 0] +%endif + punpcklbw m2, m4, m3 + pmaddubsw m4, m9, m2 + pmulhrsw m4, m8 + packsswb m4, m4 + pand m4, m10 + pandn m2, m10, m3 + por m3, m4, m2 + pxor m4, m4 + pcmpgtb m4, m3 + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add srcq, r2mp + ; lumaq has already been incremented above +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif +%endif + add grain_lutq, 82 + dec hw + jg %%loop_y_h_overlap + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif +%if %2 == 0 + xor dword r8m, 4 + ; adjust top_offxyd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + add offxyd, 16 +%endif + + ; r8m = sbym + test dword r8m, 2 +%if %2 + jne %%loop_x_hv_overlap + jmp %%loop_x_h_overlap +%else + jne %%loop_x_odd_v_overlap + jmp %%loop_x_odd +%endif + +%%end: + RET + +%%vertical_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap +%endif + + or overlapd, 2 ; top_overlap: overlap & 2 + mov r8m, overlapd + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul tmpd, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add tmpd, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and tmpd, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, tmpd +%if ARCH_X86_32 + xor sbyd, seed ; (cur_seed << 16) | top_seed + + DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%if %3 + shl r10mp, 1 +%endif +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + tmp, unused2, see, overlap, unused3, src_bak, lstride, luma_bak + + mov lstrideq, r10mp +%endif + + mov lumaq, r9mp + lea src_bakq, [srcq+wq] + lea luma_bakq, [lumaq+wq*(1+%2)] + neg wq + sub r0mp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r11m, luma_bakq + mov r4m, wq + + DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 +%else + mov r11mp, src_bakq + mov r12mp, strideq +%endif + +%%loop_x_v_overlap: +%if ARCH_X86_32 + mov seed, r3m + xor tmpd, tmpd +%endif + ; we assume from the block above that bits 8-15 of tmpd are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, h, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, top_offxy, unused, lstride + + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS tmp, src, scaling, offxy, h, picptr, top_offxy +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, top_offxy, unused, lstride, luma_bak +%endif + + movzx top_offxyd, offxyw + shr offxyd, 16 +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut +%endif + +%%loop_x_odd_v_overlap: + mov hd, r7m + mov grain_lutq, grain_lutmp +%if ARCH_X86_32 + mov r5, r5m + mova m1, [base+pb_27_17] +%else + mova m1, [pb_27_17] +%endif +%%loop_y_v_overlap: +%if ARCH_X86_32 + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq, r0, r5 + vpgatherdw m5, m6, scalingq, r0, r5 +%else + vpgatherdw m7, m4, scalingq, r12, r2 + vpgatherdw m5, m6, scalingq, r12, r2 +%endif + pcmpeqw m4, m4 + psrlw m4, 8 + pand m7, m4 + pand m5, m4 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r0, [rsp+8*mmsize+1*gprsize] + movu m4, [grain_lutq+r0] +%else + movu m4, [grain_lutq+top_offxyq] +%endif + punpckhbw m6, m4, m3 + punpcklbw m4, m3 +%if %3 + pmaddubsw m2, m9, m6 + pmaddubsw m3, m9, m4 +%else + pmaddubsw m2, m1, m6 + pmaddubsw m3, m1, m4 +%endif + pmulhrsw m2, m8 + pmulhrsw m3, m8 + packsswb m3, m2 + pxor m6, m6 + pcmpgtb m6, m3 + punpcklbw m2, m3, m6 + punpckhbw m3, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + + ; unpack chroma_source + pxor m4, m4 + punpckhbw m6, m0, m4 + punpcklbw m0, m4 ; m0-1: src as word + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m6, m3 + pmaxsw m0, m13 + pmaxsw m6, m13 + pminsw m0, m12 + pminsw m6, m12 + packuswb m0, m6 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + + dec hw + je %%end_y_v_overlap +%if ARCH_X86_32 + add srcq, r2mp + ; lumaq has already been incremented above +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif +%endif + add grain_lutq, 82 +%if %3 == 0 + btc hd, 16 +%if ARCH_X86_32 + mov r5, r5m + mova m1, [base+pb_17_27] +%else + mova m1, [pb_17_27] +%endif + jnc %%loop_y_v_overlap +%endif + jmp %%loop_y + +%%end_y_v_overlap: +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif + +%if %2 + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap +%else +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + btc dword r8m, 2 + jnc %%loop_x_odd_v_overlap +%endif + +%%loop_x_hv_overlap: +%if ARCH_X86_32 + DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, unused + + mov r6, [rsp+8*mmsize+1*gprsize] +%if %2 + lea r0, [r3d+16] + add r6, 16 + mov [rsp+8*mmsize+0*gprsize], r0 ; left_offxy +%else + mov [rsp+8*mmsize+0*gprsize], r3 ; left_offxy +%endif + mov [rsp+8*mmsize+2*gprsize], r6 ; topleft_offxy + + DEFINE_ARGS tmp, src, scaling, see, w, picptr, unused + + mov seed, r3m + xor tmpd, tmpd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + tmp, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride + +%if %2 + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offxyq+16] +%else + mov topleft_offxyq, top_offxyq + mov left_offxyq, offxyq +%endif + + ; we assume from the block above that bits 8-15 of tmpd are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS tmp, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride + + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride, luma_bak +%endif + + movzx top_offxyd, offxyw + shr offxyd, 16 +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%if ARCH_X86_32 + mov r5, r5m + mova m3, [base+pb_27_17] +%else + mova m3, [pb_27_17] +%endif +%%loop_y_hv_overlap: + ; src +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq, r0, r5 + vpgatherdw m5, m6, scalingq, r0, r5 +%else + movd m1, [grain_lutq+topleft_offxyq] +%if %3 + vpgatherdw m7, m4, scalingq, r2, r12 + vpgatherdw m5, m6, scalingq, r2, r12 +%else + vpgatherdw m7, m4, scalingq, r2, r13 + vpgatherdw m5, m6, scalingq, r2, r13 +%endif +%endif + pcmpeqw m2, m2 + psrlw m2, 8 + pand m7, m2 + pand m5, m2 + + ; grain = grain_lut[offy+y][offx+x] +%if ARCH_X86_32 + mov r0, [rsp+8*mmsize+2*gprsize] ; topleft_offxy + mov r5, [rsp+8*mmsize+1*gprsize] ; top_offxy + movd m1, [grain_lutq+r0] + mov r0, [rsp+8*mmsize+0*gprsize] ; left_offxy +%endif + movu m2, [grain_lutq+offxyq] +%if ARCH_X86_32 + movu m6, [grain_lutq+r5] + movd m4, [grain_lutq+r0] +%else + movu m6, [grain_lutq+top_offxyq] + movd m4, [grain_lutq+left_offxyq] +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m1, m6 + punpcklbw m4, m2 +%if %2 + punpcklwd m4, m1 +%else + punpckldq m4, m1 +%endif + pmaddubsw m1, m9, m4 + pmulhrsw m1, m8 + packsswb m1, m1 + pandn m4, m10, m2 + pandn m2, m10, m6 + psrldq m6, m1, 2-%2 + pand m1, m10 + pand m6, m10 + por m4, m1 + por m2, m6 + ; followed by v interpolation (top | cur -> cur) + punpckhbw m1, m2, m4 + punpcklbw m2, m4 +%if %3 + pmaddubsw m4, m9, m1 + pmaddubsw m1, m9, m2 +%else + pmaddubsw m4, m3, m1 + pmaddubsw m1, m3, m2 +%endif + pmulhrsw m4, m8 + pmulhrsw m1, m8 + packsswb m1, m4 + pxor m4, m4 + pcmpgtb m4, m1 + punpcklbw m2, m1, m4 + punpckhbw m1, m4 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m1, m5 + pmulhrsw m2, m11 + pmulhrsw m1, m11 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; unpack chroma source + pxor m4, m4 + punpckhbw m5, m0, m4 + punpcklbw m0, m4 ; m0-1: src as word + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m5, m1 + pmaxsw m0, m13 + pmaxsw m5, m13 + pminsw m0, m12 + pminsw m5, m12 + packuswb m0, m5 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add srcq, r2mp + ; lumaq has been adjusted above already +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*(1+%2)] +%else + add lumaq, r10mp +%endif +%endif + add grain_lutq, 82 + dec hw +%if %3 + jg %%loop_y_h_overlap +%else + jle %%end_y_hv_overlap +%if ARCH_X86_32 + mov r5, r5m + mova m3, [base+pb_17_27] +%else + mova m3, [pb_17_27] +%endif + btc hd, 16 + jnc %%loop_y_hv_overlap +%if ARCH_X86_64 + mov lstrideq, r10mp +%endif + jmp %%loop_y_h_overlap +%%end_y_hv_overlap: +%if ARCH_X86_64 + mov lstrideq, r10mp +%endif +%endif + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif +%if %2 + jmp %%loop_x_hv_overlap +%else +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + xor dword r8m, 4 + jmp %%loop_x_odd_v_overlap +%endif + +%%end_hv: + RET +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +%endmacro + +FGUV_FN 420, 1, 1 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif + +FGUV_FN 422, 1, 0 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif + +FGUV_FN 444, 0, 0 diff --git a/third_party/dav1d/src/x86/ipred.asm b/third_party/dav1d/src/x86/ipred.asm --- a/third_party/dav1d/src/x86/ipred.asm +++ b/third_party/dav1d/src/x86/ipred.asm @@ -98,10 +98,12 @@ db 2, 3, 2, 3, 6, 7, 6, 7, 10, 11, 10, 11, 14, 15, 14, 15 ipred_h_shuf: db 7, 7, 7, 7, 3, 3, 3, 3, 5, 5, 5, 5, 1, 1, 1, 1 db 6, 6, 6, 6, 2, 2, 2, 2, 4, 4, 4, 4; 0, 0, 0, 0 pw_64: times 2 dw 64 +cfl_ac_444_w16_pad1_shuffle: db 0, -1, 1, -1, 2, -1, 3, -1, 4, -1, 5, -1, 6, -1 + times 9 db 7, -1 cfl_ac_w16_pad_shuffle: ; w=16, w_pad=1 db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ; w=8, w_pad=1 as well as second half of previous one cfl_ac_w8_pad1_shuffle: db 0, 1, 2, 3, 4, 5 times 5 db 6, 7 @@ -164,10 +166,11 @@ JMP_TABLE ipred_cfl, avx2, h4, h8, h16, h32, w4, w8, w16, w32, \ s4-8*4, s8-8*4, s16-8*4, s32-8*4 JMP_TABLE ipred_cfl_left, avx2, h4, h8, h16, h32 JMP_TABLE ipred_cfl_ac_420, avx2, w16_pad1, w16_pad2, w16_pad3 JMP_TABLE ipred_cfl_ac_422, avx2, w16_pad1, w16_pad2, w16_pad3 +JMP_TABLE ipred_cfl_ac_444, avx2, w32_pad1, w32_pad2, w32_pad3, w4, w8, w16, w32 JMP_TABLE pal_pred, avx2, w4, w8, w16, w32, w64 cextern dr_intra_derivative cextern filter_intra_taps @@ -306,11 +309,11 @@ cmp hd, 8 je .w8_end mov r6d, 0x5556 mov r2d, 0x3334 cmp hd, 32 - cmovz r6d, r2d + cmove r6d, r2d movd xm1, r6d pmulhuw xm0, xm1 .w8_end: vpbroadcastb xm0, xm0 .s8: @@ -1439,11 +1442,11 @@ paddw m0, m2 pmulhrsw m0, m3 mov r3d, 9 mov tlq, rsp cmp hd, 4 - cmova maxbased, r3d + cmovne maxbased, r3d vextracti128 xm1, m0, 1 packuswb xm0, xm1 mova [tlq], xm0 .w4_main: movd xm6, dxd @@ -1626,12 +1629,12 @@ .w8_3tap: pmulhrsw m1, m3 sar r5d, 1 mov tlq, rsp add r5d, 17 ; w*2 + (filter_strength == 3) - cmp hd, 8 - cmova maxbased, r5d + cmp hd, 16 + cmovns maxbased, r5d mov [tlq+r5], r3b vextracti128 xm0, m1, 1 packuswb xm0, xm1 mova [tlq], xm0 .w8_main: @@ -1743,12 +1746,12 @@ pmulhrsw m0, m3 pmulhrsw m1, m3 sar r5d, 1 mov tlq, rsp add r5d, 33 - cmp hd, 16 - cmova maxbased, r5d + cmp hd, 32 + cmovns maxbased, r5d mov [tlq+r5], r3b packuswb m0, m1 vpermq m0, m0, q3120 mova [tlq], m0 .w16_main: @@ -1810,11 +1813,11 @@ %assign stack_offset org_stack_offset ALLOC_STACK -96, 15 lea r3d, [hq+31] mov maxbased, 63 cmp hd, 32 - cmovb maxbased, r3d + cmovs maxbased, r3d test angled, 0x400 ; !enable_intra_edge_filter jnz .w32_main vbroadcasti128 m0, [pb_0to15] sub r3d, 29 ; h+2 movu xm13, [tlq+29] ; 32-39 @@ -1887,12 +1890,12 @@ shr r2d, 3 mov [rsp+64], r2b mov tlq, rsp mov [tlq+65], r3b mov r3d, 65 - cmp hd, 32 - cmova maxbased, r3d + cmp hd, 64 + cmove maxbased, r3d packuswb m0, m2 packuswb m1, m6 mova [tlq+ 0], m0 mova [tlq+32], m1 .w32_main: @@ -2292,11 +2295,11 @@ jz .w4_main mov r5d, 10 cmp hd, 16 movu xm2, [rsp+49] vinserti128 m2, [rsp+43], 1 - cmovl r5d, hd + cmovs r5d, hd xor r5d, 15 ; h == 16 ? 5 : 15 - h movd xm0, r5d vbroadcasti128 m1, [base+z_filter_s+12] vbroadcasti128 m4, [base+z_filter_s+16] vinserti128 m3, m1, [z_filter_s+8], 1 ; 56 67 78 89 9a ab bc cd 55 55 56 67 78 89 9a ab @@ -2499,11 +2502,11 @@ pmulhrsw m3, m13 jmp .w8_filter_left_top16 .w8_filter_left_h16: mov r5d, 10 cmp hd, 16 - cmovl r5d, hd + cmovs r5d, hd xor r5d, 15 ; h == 16 ? 5 : 15 - h movd xm0, r5d vpbroadcastb m0, xm0 .w8_filter_left_top16: vbroadcasti128 m1, [base+z_filter_s+12] @@ -2740,11 +2743,11 @@ pmulhrsw m3, m13 jmp .w16_filter_left_top16 .w16_filter_left_h16: mov r5d, 10 cmp hd, 16 - cmovl r5d, hd + cmovs r5d, hd xor r5d, 15 ; h == 16 ? 5 : 15 - h movd xm0, r5d vpbroadcastb m0, xm0 .w16_filter_left_top16: movu xm2, [tlq-15] @@ -3113,11 +3116,11 @@ paddw m0, m2 pmulhrsw m0, m3 mov r4d, 9 lea tlq, [rsp+15] cmp wd, 4 - cmova maxbased, r4d + cmovne maxbased, r4d vextracti128 xm1, m0, 1 packuswb xm0, xm1 mova [rsp], xm0 .h4_main: movd xm6, dyd @@ -3319,12 +3322,12 @@ .h8_3tap: pmulhrsw m0, m3 sar r5d, 1 lea tlq, [rsp+31] add r5d, 17 - cmp wd, 8 - cmova maxbased, r5d + cmp wd, 16 + cmovns maxbased, r5d neg r5 mov [tlq+r5], r4b vextracti128 xm1, m0, 1 packuswb xm0, xm1 mova [tlq-15], xm0 @@ -3383,11 +3386,11 @@ .h8_transpose: mova xm2, [rsp+16*1] sub org_wd, 8 lea r2, [strideq*3] lea r6, [dstq+org_wq] - cmovg dstq, r6 + cmovns dstq, r6 punpcklwd xm1, xm2, xm0 punpckhwd xm2, xm0 lea r6, [dstq+strideq*4] jge .h8_w8 add rsp, 16*2 @@ -3491,12 +3494,12 @@ pmulhrsw m0, m3 pmulhrsw m1, m3 sar r5d, 1 lea tlq, [rsp+63] add r5d, 33 - cmp wd, 16 - cmova maxbased, r5d + cmp wd, 32 + cmovns maxbased, r5d neg r5 mov [tlq+r5], r4b packuswb m0, m1 vpermq m0, m0, q2031 mova [tlq-31], m0 @@ -3561,11 +3564,11 @@ .h16_transpose: mova m2, [rsp+32*1] sub org_wd, 8 lea r2, [strideq*3] lea r6, [dstq+org_wq] - cmovg dstq, r6 + cmovns dstq, r6 punpcklbw m1, m2, m0 punpckhbw m2, m0 lea r3, [strideq*5] punpcklbw m0, m1, m2 punpckhbw m1, m2 @@ -3650,11 +3653,11 @@ mov r4d, 21 mov r5d, 3 movu xm11, [tlq-66] ; 56-63 vinserti128 m11, [tlq-52], 1 ; 40-47 sub r4d, wd ; 21-w - cmovg r5d, r4d + cmovns r5d, r4d movu xm12, [tlq-58] ; 48-55 vinserti128 m12, [tlq-44], 1 ; 32-39 sub r4d, 8 ; 13-w movd xm1, r5d movu xm13, [tlq-34] ; 24-31 @@ -3719,12 +3722,12 @@ shr r2d, 3 mov [rsp+31], r2b lea tlq, [rsp+95] mov [tlq-65], r4b mov r4d, 65 - cmp wd, 32 - cmova maxbased, r4d + cmp wd, 64 + cmove maxbased, r4d packuswb m0, m2 packuswb m1, m6 mova [tlq-63], m0 mova [tlq-31], m1 .h32_main: @@ -4551,11 +4554,11 @@ cmp hd, 8 je .w8_end mov r6d, 0x5556 mov r2d, 0x3334 cmp hd, 32 - cmovz r6d, r2d + cmove r6d, r2d movd xm1, r6d pmulhuw xm0, xm1 .w8_end: vpbroadcastw m0, xm0 .s8: @@ -5052,10 +5055,240 @@ add ac_bakq, 32 sub szd, 16 jg .sub_loop RET +cglobal ipred_cfl_ac_444, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak + movifnidn hpadd, hpadm + movifnidn wd, wm + mov hd, hm + mov szd, wd + imul szd, hd + shl hpadd, 2 + sub hd, hpadd + pxor m4, m4 + vpbroadcastd m5, [pw_1] + tzcnt r8d, wd + lea r5, [ipred_cfl_ac_444_avx2_table] + movsxd r8, [r5+r8*4+12] + add r5, r8 + + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, sz, ac_bak + mov ac_bakq, acq + jmp r5 + +.w4: + lea stride3q, [strideq*3] + pxor xm2, xm2 +.w4_loop: + movd xm1, [yq] + movd xm0, [yq+strideq*2] + pinsrd xm1, [yq+strideq], 1 + pinsrd xm0, [yq+stride3q], 1 + punpcklbw xm1, xm2 + punpcklbw xm0, xm2 + psllw xm1, 3 + psllw xm0, 3 + mova [acq], xm1 + mova [acq+16], xm0 + paddw xm1, xm0 + paddw xm4, xm1 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg_mul + pshufd xm0, xm0, q3232 + paddw xm1, xm0, xm0 +.w4_hpad_loop: + mova [acq], xm0 + mova [acq+16], xm0 + paddw xm4, xm1 + add acq, 32 + sub hpadd, 4 + jg .w4_hpad_loop + jmp .calc_avg_mul + +.w8: + lea stride3q, [strideq*3] + pxor m2, m2 +.w8_loop: + movq xm1, [yq] + movq xm0, [yq+strideq*2] + vinserti128 m1, [yq+strideq], 1 + vinserti128 m0, [yq+stride3q], 1 + punpcklbw m1, m2 + punpcklbw m0, m2 + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + paddw m4, m1 + lea yq, [yq+strideq*4] + add acq, 64 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg_mul + vpermq m0, m0, q3232 + paddw m1, m0, m0 +.w8_hpad_loop: + mova [acq], m0 + mova [acq+32], m0 + paddw m4, m1 + add acq, 64 + sub hpadd, 4 + jg .w8_hpad_loop + jmp .calc_avg_mul + +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + pmovzxbw m1, [yq] + pmovzxbw m0, [yq+strideq] + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + pmaddwd m1, m5 + paddd m4, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_loop + test hpadd, hpadd + jz .calc_avg + jmp .w16_hpad +.w16_wpad: + mova m3, [cfl_ac_444_w16_pad1_shuffle] +.w16_wpad_loop: + vpbroadcastq m1, [yq] + vpbroadcastq m0, [yq+strideq] + pshufb m1, m3 + pshufb m0, m3 + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + pmaddwd m1, m5 + paddd m4, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_wpad_loop + test hpadd, hpadd + jz .calc_avg +.w16_hpad: + paddw m1, m0, m0 + pmaddwd m1, m5 +.w16_hpad_loop: + mova [acq], m0 + mova [acq+32], m0 + paddd m4, m1 + add acq, 64 + sub hpadd, 2 + jg .w16_hpad_loop + jmp .calc_avg + +.w32: + test wpadd, wpadd + jnz .w32_wpad +.w32_loop: + pmovzxbw m1, [yq] + pmovzxbw m0, [yq+16] + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m2, m1, m0 + pmaddwd m2, m5 + paddd m4, m2 + add yq, strideq + add acq, 64 + dec hd + jg .w32_loop + test hpadd, hpadd + jz .calc_avg + jmp .w32_hpad_loop +.w32_wpad: + DEFINE_ARGS ac, y, stride, wpad, hpad, iptr, h, sz, ac_bak + lea iptrq, [ipred_cfl_ac_444_avx2_table] + add wpadd, wpadd + mova m3, [iptrq+cfl_ac_444_w16_pad1_shuffle-ipred_cfl_ac_444_avx2_table] + movsxd wpadq, [iptrq+wpadq+4] + add iptrq, wpadq + jmp iptrq +.w32_pad3: + vpbroadcastq m1, [yq] + pshufb m1, m3 + vpermq m0, m1, q3232 + jmp .w32_wpad_end +.w32_pad2: + pmovzxbw m1, [yq] + pshufhw m0, m1, q3333 + vpermq m0, m0, q3333 + jmp .w32_wpad_end +.w32_pad1: + pmovzxbw m1, [yq] + vpbroadcastq m0, [yq+16] + pshufb m0, m3 + ; fall-through +.w32_wpad_end: + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m2, m1, m0 + pmaddwd m2, m5 + paddd m4, m2 + add yq, strideq + add acq, 64 + dec hd + jz .w32_wpad_done + jmp iptrq +.w32_wpad_done: + test hpadd, hpadd + jz .calc_avg +.w32_hpad_loop: + mova [acq], m1 + mova [acq+32], m0 + paddd m4, m2 + add acq, 64 + dec hpadd + jg .w32_hpad_loop + jmp .calc_avg + +.calc_avg_mul: + pmaddwd m4, m5 +.calc_avg: + vextracti128 xm1, m4, 1 + tzcnt r1d, szd + paddd xm0, xm4, xm1 + movd xm2, r1d + movd xm3, szd + punpckhqdq xm1, xm0, xm0 + paddd xm0, xm1 + psrad xm3, 1 + psrlq xm1, xm0, 32 + paddd xm0, xm3 + paddd xm0, xm1 + psrad xm0, xm2 + vpbroadcastw m0, xm0 +.sub_loop: + mova m1, [ac_bakq] + psubw m1, m0 + mova [ac_bakq], m1 + add ac_bakq, 32 + sub szd, 16 + jg .sub_loop + RET + cglobal pal_pred, 4, 6, 5, dst, stride, pal, idx, w, h vbroadcasti128 m4, [palq] lea r2, [pal_pred_avx2_table] tzcnt wd, wm movifnidn hd, hm diff --git a/third_party/dav1d/src/x86/ipred_init_tmpl.c b/third_party/dav1d/src/x86/ipred_init_tmpl.c --- a/third_party/dav1d/src/x86/ipred_init_tmpl.c +++ b/third_party/dav1d/src/x86/ipred_init_tmpl.c @@ -48,10 +48,11 @@ decl_cfl_pred_fn(dav1d_ipred_cfl_top_avx2); decl_cfl_pred_fn(dav1d_ipred_cfl_left_avx2); decl_cfl_ac_fn(dav1d_ipred_cfl_ac_420_avx2); decl_cfl_ac_fn(dav1d_ipred_cfl_ac_422_avx2); +decl_cfl_ac_fn(dav1d_ipred_cfl_ac_444_avx2); decl_pal_pred_fn(dav1d_pal_pred_avx2); decl_angular_ipred_fn(dav1d_ipred_dc_ssse3); decl_angular_ipred_fn(dav1d_ipred_dc_128_ssse3); @@ -129,9 +130,10 @@ c->cfl_pred[TOP_DC_PRED] = dav1d_ipred_cfl_top_avx2; c->cfl_pred[LEFT_DC_PRED] = dav1d_ipred_cfl_left_avx2; c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_ipred_cfl_ac_420_avx2; c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_ipred_cfl_ac_422_avx2; + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_ipred_cfl_ac_444_avx2; c->pal_pred = dav1d_pal_pred_avx2; #endif } diff --git a/third_party/dav1d/src/x86/itx.asm b/third_party/dav1d/src/x86/itx.asm --- a/third_party/dav1d/src/x86/itx.asm +++ b/third_party/dav1d/src/x86/itx.asm @@ -58,11 +58,10 @@ pw_8192: times 2 dw 8192 pw_16384: times 2 dw 16384 pw_1697x16: times 2 dw 1697*16 pw_1697x8: times 2 dw 1697*8 pw_2896x8: times 2 dw 2896*8 -pw_5793x4: times 2 dw 5793*4 pd_2048: dd 2048 COEF_PAIR 2896, 2896 COEF_PAIR 1567, 3784 @@ -391,11 +390,11 @@ %ifidn %1_%2, dct_identity vpbroadcastd m0, [o(pw_2896x8)] pmulhrsw m0, [cq] vpbroadcastd m1, [o(pw_1697x8)] pmulhrsw m1, m0 - paddw m0, m1 + paddsw m0, m1 punpcklwd m0, m0 punpckhdq m1, m0, m0 punpckldq m0, m0 jmp m(iadst_4x4_internal).end %elifidn %1_%2, identity_dct @@ -403,11 +402,11 @@ packusdw m0, [cq+16*1] vpbroadcastd m1, [o(pw_1697x8)] vpbroadcastd m2, [o(pw_2896x8)] packusdw m0, m0 pmulhrsw m1, m0 - paddw m0, m1 + paddsw m0, m1 pmulhrsw m0, m2 mova m1, m0 jmp m(iadst_4x4_internal).end %elif %3 >= 0 vpbroadcastw m0, [cq] @@ -559,23 +558,23 @@ mova m0, [cq+16*0] mova m1, [cq+16*1] vpbroadcastd m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 - paddw m0, m2 - paddw m1, m3 + paddsw m0, m2 + paddsw m1, m3 punpckhwd m2, m0, m1 punpcklwd m0, m1 punpckhwd m1, m0, m2 punpcklwd m0, m2 jmp tx2q .pass2: vpbroadcastd m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 - paddw m0, m2 - paddw m1, m3 + paddsw m0, m2 + paddsw m1, m3 jmp m(iadst_4x4_internal).end %macro WRITE_4X8 2 ; coefs[1-2] movd xm4, [dstq+strideq*0] pinsrd xm4, [dstq+strideq*1], 1 @@ -624,11 +623,11 @@ vpbroadcastd xm3, [o(pw_1697x8)] vpbroadcastd xm4, [o(pw_2048)] punpckldq xm0, xm1 pmulhrsw xm0, xm2 pmulhrsw xm3, xm0 - paddw xm0, xm3 + paddsw xm0, xm3 pmulhrsw xm0, xm2 pmulhrsw xm0, xm4 vpbroadcastq m0, xm0 mova m1, m0 jmp m(iadst_4x8_internal).end3 @@ -905,12 +904,12 @@ pmulhrsw m2, m3 punpcklwd m0, m1, m2 punpckhwd m1, m2 pmulhrsw m2, m4, m0 pmulhrsw m4, m1 - paddw m0, m2 - paddw m1, m4 + paddsw m0, m2 + paddsw m1, m4 jmp tx2q .pass2: vpbroadcastd m4, [o(pw_4096)] jmp m(iadst_4x8_internal).end2 @@ -923,12 +922,12 @@ vpbroadcastd m1, [o(pw_16384)] vpbroadcastd m2, [o(pw_1697x16)] vpbroadcastd m3, [o(pw_2048)] pmulhrsw m0, m1 pmulhrsw m2, m0 - paddw m0, m0 - paddw m0, m2 + paddsw m0, m0 + paddsw m0, m2 pmulhrsw m3, m0 punpcklwd m1, m3, m3 punpckhwd m3, m3 punpckldq m0, m1, m1 punpckhdq m1, m1 @@ -939,19 +938,20 @@ movd xm0, [cq+32*0] punpcklwd xm0, [cq+32*1] movd xm1, [cq+32*2] punpcklwd xm1, [cq+32*3] vpbroadcastd xm2, [o(pw_1697x8)] - vpbroadcastd xm3, [o(pw_16384)] - vpbroadcastd xm4, [o(pw_2896x8)] + vpbroadcastd xm3, [o(pw_2896x8)] + vpbroadcastd xm4, [o(pw_2048)] punpckldq xm0, xm1 + pcmpeqw xm1, xm1 pmulhrsw xm2, xm0 - paddw xm0, xm2 + pcmpeqw xm1, xm0 + pxor xm0, xm1 + pavgw xm0, xm2 pmulhrsw xm0, xm3 - psrlw xm3, 3 ; pw_2048 pmulhrsw xm0, xm4 - pmulhrsw xm0, xm3 vpbroadcastq m0, xm0 mova m1, m0 mova m2, m0 mova m3, m0 jmp m(iadst_4x16_internal).end3 @@ -1281,43 +1281,50 @@ cglobal iidentity_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 mova m3, [cq+32*0] mova m2, [cq+32*1] mova m4, [cq+32*2] - mova m0, [cq+32*3] - vpbroadcastd m5, [o(pw_1697x8)] + mova m5, [cq+32*3] + vpbroadcastd m8, [o(pw_1697x8)] + pcmpeqw m0, m0 ; -1 punpcklwd m1, m3, m2 punpckhwd m3, m2 - punpcklwd m2, m4, m0 - punpckhwd m4, m0 - pmulhrsw m0, m5, m1 - pmulhrsw m6, m5, m2 - pmulhrsw m7, m5, m3 - pmulhrsw m5, m4 - paddw m1, m0 - paddw m2, m6 - paddw m3, m7 - paddw m4, m5 - vpbroadcastd m5, [o(pw_16384)] + punpcklwd m2, m4, m5 + punpckhwd m4, m5 + pmulhrsw m5, m8, m1 + pmulhrsw m6, m8, m2 + pmulhrsw m7, m8, m3 + pmulhrsw m8, m4 + pcmpeqw m9, m0, m1 ; we want to do a signed avg, but pavgw is + pxor m1, m9 ; unsigned. as long as both signs are equal + pcmpeqw m9, m0, m2 ; it still works, but if the input is -1 the + pxor m2, m9 ; pmulhrsw result will become 0 which causes + pcmpeqw m9, m0, m3 ; pavgw to output -32768 instead of 0 unless + pxor m3, m9 ; we explicitly deal with that case here. + pcmpeqw m0, m4 + pxor m4, m0 + pavgw m1, m5 + pavgw m2, m6 + pavgw m3, m7 + pavgw m4, m8 punpckldq m0, m1, m2 punpckhdq m1, m2 punpckldq m2, m3, m4 punpckhdq m3, m4 - REPX {pmulhrsw x, m5}, m0, m1, m2, m3 jmp tx2q .pass2: vpbroadcastd m8, [o(pw_1697x16)] vpbroadcastd m5, [o(pw_2048)] pmulhrsw m4, m8, m0 pmulhrsw m6, m8, m1 pmulhrsw m7, m8, m2 pmulhrsw m8, m3 - REPX {paddw x, x}, m0, m1, m2, m3 - paddw m0, m4 - paddw m1, m6 - paddw m2, m7 - paddw m3, m8 + REPX {paddsw x, x}, m0, m1, m2, m3 + paddsw m0, m4 + paddsw m1, m6 + paddsw m2, m7 + paddsw m3, m8 jmp m(iadst_4x16_internal).end2 %macro WRITE_8X4 4-7 strideq*1, strideq*2, r3, ; coefs[1-2], tmp[1-2], off[1-3] movq xm%3, [dstq ] movhps xm%3, [dstq+%5] @@ -1351,11 +1358,11 @@ pmulhrsw xm1, xm0, [cq] vpbroadcastd xm2, [o(pw_1697x8)] vpbroadcastd xm3, [o(pw_2048)] pmulhrsw xm1, xm0 pmulhrsw xm2, xm1 - paddw xm1, xm2 + paddsw xm1, xm2 pmulhrsw xm1, xm3 punpcklwd xm1, xm1 punpckldq xm0, xm1, xm1 punpckhdq xm1, xm1 vpermq m0, m0, q1100 @@ -1367,11 +1374,11 @@ packusdw xm1, [cq+16*3] vpbroadcastd xm2, [o(pw_2896x8)] vpbroadcastd xm3, [o(pw_2048)] packusdw xm0, xm1 pmulhrsw xm0, xm2 - paddw xm0, xm0 + paddsw xm0, xm0 pmulhrsw xm0, xm2 pmulhrsw xm0, xm3 vinserti128 m0, m0, xm0, 1 mova m1, m0 %else @@ -1445,11 +1452,11 @@ vinserti128 m0, m0, xm2, 1 vinserti128 m1, m1, xm3, 1 punpckhwd m2, m0, m1 punpcklwd m0, m1 pxor m3, m3 - psubw m3, m2 + psubsw m3, m2 punpckhwd m1, m0, m3 punpcklwd m0, m3 jmp tx2q .pass2: call .main @@ -1490,11 +1497,11 @@ vinserti128 m3, m3, xm1, 1 vinserti128 m2, m2, xm0, 1 punpckhwd m1, m3, m2 punpcklwd m3, m2 pxor m0, m0 - psubw m0, m1 + psubsw m0, m1 punpckhwd m1, m0, m3 punpcklwd m0, m3 jmp tx2q .pass2: call m(iadst_8x4_internal).main @@ -1518,19 +1525,19 @@ punpckhwd m2, m0 pmulhrsw m1, m3 pmulhrsw m2, m3 punpcklwd m0, m1, m2 punpckhwd m1, m2 - paddw m0, m0 - paddw m1, m1 + paddsw m0, m0 + paddsw m1, m1 jmp tx2q .pass2: vpbroadcastd m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 - paddw m0, m2 - paddw m1, m3 + paddsw m0, m2 + paddsw m1, m3 jmp m(iadst_8x4_internal).end %macro INV_TXFM_8X8_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 8x8 %ifidn %1_%2, dct_identity @@ -1794,12 +1801,12 @@ mova [cq], m3 pmulhrsw m7, m0 pmulhrsw m7, m1 psrlw m1, 3 ; pw_2048 pmulhrsw m2, m7 - paddw m7, m7 - paddw m7, m2 + paddsw m7, m7 + paddsw m7, m2 pmulhrsw m7, m1 punpcklwd m5, m7, m7 punpckhwd m7, m7 punpcklwd m4, m5, m5 punpckhwd m5, m5 @@ -2118,16 +2125,16 @@ INV_TXFM_8X16_FN identity, flipadst INV_TXFM_8X16_FN identity, identity %macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16394] pmulhrsw m%2, m%3, m%1 -%if %0 == 4 ; if we're going to downshift by 1 doing so here eliminates the paddw +%if %0 == 4 ; if downshifting by 1 pmulhrsw m%2, m%4 %else - paddw m%1, m%1 + paddsw m%1, m%1 %endif - paddw m%1, m%2 + paddsw m%1, m%2 %endmacro cglobal iidentity_8x16_internal, 0, 5, 13, dst, stride, c, eob, tx2 mova xm3, [cq+16*0] mova xm2, [cq+16*2] @@ -2199,11 +2206,11 @@ vpbroadcastd xm0, [o(pw_16384)] vpbroadcastd xm1, [o(pw_1697x8)] pmulhrsw xm3, xm0 psrlw xm0, 3 ; pw_2048 pmulhrsw xm1, xm3 - paddw xm3, xm1 + paddsw xm3, xm1 pmulhrsw xm3, xm0 punpcklwd xm3, xm3 punpckldq xm1, xm3, xm3 punpckhdq xm3, xm3 vpbroadcastq m0, xm1 @@ -2226,11 +2233,11 @@ packusdw m1, m3 packusdw m0, m1 vpbroadcastd m1, [o(pw_2896x8)] pmulhrsw m4, m0 pmulhrsw m4, m5 - paddw m0, m4 + paddsw m0, m4 psrlw m5, 3 ; pw_2048 pmulhrsw m0, m1 pmulhrsw m0, m5 mov r3d, 2 .end: @@ -2501,14 +2508,14 @@ pmulhrsw m0, m7, m1 pmulhrsw m5, m7, m2 pmulhrsw m6, m7, m3 pmulhrsw m7, m4 REPX {pmulhrsw x, m8}, m0, m5, m6, m7 - paddw m1, m0 - paddw m2, m5 - paddw m3, m6 - paddw m4, m7 + paddsw m1, m0 + paddsw m2, m5 + paddsw m3, m6 + paddsw m4, m7 punpcklqdq m0, m1, m2 punpckhqdq m1, m2 punpcklqdq m2, m3, m4 punpckhqdq m3, m4 jmp tx2q @@ -2516,14 +2523,14 @@ vpbroadcastd m7, [o(pw_1697x8)] pmulhrsw m4, m7, m0 pmulhrsw m5, m7, m1 pmulhrsw m6, m7, m2 pmulhrsw m7, m3 - paddw m0, m4 - paddw m1, m5 - paddw m2, m6 - paddw m3, m7 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 jmp m(iadst_16x4_internal).end %macro INV_TXFM_16X8_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 16x8 %ifidn %1_%2, dct_dct @@ -2579,11 +2586,11 @@ vpermq m0, m0, q1100 punpcklwd m0, m1 pmulhrsw m0, m4 pmulhrsw m5, m0 pmulhrsw m5, m2 - paddw m0, m5 + paddsw m0, m5 psrlw m2, 3 ; pw_2048 pmulhrsw m0, m4 pmulhrsw m0, m2 mov r3d, 4 jmp m(inv_txfm_add_identity_dct_16x4).end @@ -2901,20 +2908,19 @@ %elifidn %1_%2, dct_identity WIN64_SPILL_XMM 7 vpbroadcastd m3, [o(pw_2896x8)] pmulhrsw m3, [cq] vpbroadcastd m0, [o(pw_8192)] - vpbroadcastd m1, [o(pw_5793x4)] + vpbroadcastd m1, [o(pw_1697x16)] vpbroadcastw m4, [o(deint_shuf)] ; pb_0_1 pcmpeqb m5, m5 pxor m6, m6 mova [cq], m6 paddb m5, m5 ; pb_m2 pmulhrsw m3, m0 psrlw m0, 2 ; pw_2048 - psllw m3, 2 - pmulhrsw m3, m1 + IDTX16 3, 1, 1 pmulhrsw m3, m0 mov r3d, 8 .loop: mova xm1, [dstq] vinserti128 m1, m1, [dstq+strideq*8], 1 @@ -2952,21 +2958,19 @@ vinserti128 m2, m2, [cq+32*14], 1 vinserti128 m4, m4, [cq+32*15], 1 punpcklwd m1, m3 vpbroadcastd m3, [o(pw_1697x16)] punpcklwd m2, m4 - vpbroadcastd m4, [o(pw_8192)] + vpbroadcastd m4, [o(pw_2896x8)] punpckldq m1, m2 - vpbroadcastd m2, [o(pw_2896x8)] + vpbroadcastd m2, [o(pw_2048)] punpcklqdq m0, m1 pmulhrsw m3, m0 - paddw m0, m0 - paddw m0, m3 + psraw m3, 1 + pavgw m0, m3 pmulhrsw m0, m4 - psrlw m4, 2 ; pw_2048 pmulhrsw m0, m2 - pmulhrsw m0, m4 mov r3d, 8 jmp m(inv_txfm_add_identity_dct_16x4).end %endif %endmacro @@ -3383,10 +3387,16 @@ lea dstq, [dstq+strideq*4] WRITE_16X2 5, 6, 0, 1, strideq*0, strideq*1 WRITE_16X2 7, [rsp+32*2], 0, 1, strideq*2, r3 jmp m(idct_16x16_internal).end3 +%macro IDTX16B 3 ; src/dst, tmp, pw_1697x16 + pmulhrsw m%2, m%3, m%1 + psraw m%2, 1 + pavgw m%1, m%2 ; signs are guaranteed to be equal +%endmacro + INV_TXFM_16X16_FN identity, dct, 15 INV_TXFM_16X16_FN identity, identity cglobal iidentity_16x16_internal, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 vpbroadcastd m7, [o(pw_1697x16)] @@ -3417,26 +3427,21 @@ vinserti128 m12, [cq+16*11], 1 mova xm13, [cq-16* 3] vinserti128 m13, [cq+16*13], 1 mova xm14, [cq-16* 1] vinserti128 m14, [cq+16*15], 1 - REPX {IDTX16 x, 6, 7}, 0, 15, 1, 8, 2, 9, 3, \ + REPX {IDTX16B x, 6, 7}, 0, 15, 1, 8, 2, 9, 3, \ 10, 4, 11, 5, 12, 13, 14 mova xm6, [cq-16* 4] vinserti128 m6, [cq+16*12], 1 - mova [rsp], m1 - IDTX16 6, 1, 7 - mova xm1, [cq-16* 2] - vinserti128 m1, [cq+16*14], 1 - pmulhrsw m7, m1 - paddw m1, m1 - paddw m7, m1 - vpbroadcastd m1, [o(pw_8192)] - REPX {pmulhrsw x, m1}, m0, m2, m3, m4, m5, m6, m7, \ - m8, m9, m10, m11, m12, m13, m14, m15 - pmulhrsw m1, [rsp] mova [rsp], m0 + IDTX16B 6, 0, 7 + mova xm0, [cq-16* 2] + vinserti128 m0, [cq+16*14], 1 + pmulhrsw m7, m0 + psraw m7, 1 + pavgw m7, m0 jmp m(idct_16x16_internal).pass1_end3 ALIGN function_align .pass2: vpbroadcastd m15, [o(pw_1697x16)] mova [rsp+32*1], m0 @@ -3445,12 +3450,12 @@ mova m0, [rsp+32*1] mova [rsp+32*1], m1 IDTX16 0, 1, 15 mova m1, [rsp+32*0] pmulhrsw m15, m1 - paddw m1, m1 - paddw m15, m1 + paddsw m1, m1 + paddsw m15, m1 jmp m(idct_16x16_internal).end %define o_base iadst4_dconly2a + 128 %macro LOAD_8ROWS 2-3 0 ; src, stride, is_rect2 @@ -3961,11 +3966,11 @@ mova xm6, [cq+16* 8] mova xm7, [cq+16*12] vinserti128 m6, m6, [cq+16* 9], 1 vinserti128 m7, m7, [cq+16*13], 1 REPX {mova [cq+32*x], m8}, -4, -2, 0, 2, 4, 6 - REPX {paddw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 call .transpose8x8 REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_8X4 0, 4, 8, 10, strideq*8, strideq*4, r4*4 add dstq, strideq WRITE_8X4 1, 5, 0, 4, strideq*8, strideq*4, r4*4 @@ -4570,24 +4575,25 @@ mova [rsp+gprsize+32*0], m6 mova [rsp+gprsize+32*1], m7 IDCT32_PASS1_END 1, 9, 6, 7 ret -cglobal inv_txfm_add_identity_identity_16x32, 4, 5, 12, dst, stride, c, eob +cglobal inv_txfm_add_identity_identity_16x32, 4, 5, 13, dst, stride, c, eob %undef cmp lea rax, [o_base] vpbroadcastd m9, [o(pw_2896x8)] - vpbroadcastd m10, [o(pw_5793x4)] - vpbroadcastd m11, [o(pw_5)] + vpbroadcastd m10, [o(pw_1697x16)] + vpbroadcastd m12, [o(pw_8192)] cmp eobd, 43 ; if (eob > 43) setg r4b ; iteration_count++ cmp eobd, 150 ; if (eob > 150) setg al ; iteration_count++ add eobd, -279 ; if (eob > 278) adc r4b, al ; iteration_count++ lea r3, [strideq*3] mov rax, cq + paddw m11, m12, m12 ; pw_16384 .loop: mova xm0, [cq+64* 0] mova xm1, [cq+64* 1] vinserti128 m0, m0, [cq+64* 8], 1 vinserti128 m1, m1, [cq+64* 9], 1 @@ -4602,15 +4608,13 @@ mova xm6, [cq+64* 6] mova xm7, [cq+64* 7] vinserti128 m6, m6, [cq+64*14], 1 vinserti128 m7, m7, [cq+64*15], 1 REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 - REPX {psllw x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {IDTX16 x, 8, 10, 11}, 0, 1, 2, 3, 4, 5, 6, 7 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 - REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 - REPX {paddw x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 - REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r3 lea dstq, [dstq+strideq*4] WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 WRITE_16X2 6, 7, 0, 1, strideq*2, r3 @@ -4620,17 +4624,17 @@ jge .loop sub cq, 32 pxor m0, m0 mov r0d, 8 cmp cq, rax - jg .zero_loop + ja .zero_loop .zero_loop_half: mova [rax+64*0], m0 mova [rax+64*1], m0 - mova [rax+64*2], m0 - mova [rax+64*3], m0 add rax, 64*4 + mova [rax-64*2], m0 + mova [rax-64*1], m0 sub r0d, 2 jg .zero_loop_half RET .zero_loop: mova [rax+32*0], m0 @@ -4644,11 +4648,11 @@ cglobal inv_txfm_add_identity_identity_32x16, 4, 6, 12, dst, stride, c, eob %undef cmp lea rax, [o_base] vpbroadcastd m9, [o(pw_2896x8)] - vpbroadcastd m10, [o(pw_1697x8)] + vpbroadcastd m10, [o(pw_1697x16)] vpbroadcastd m11, [o(pw_2048)] cmp eobd, 35 ; if (eob > 35) setg r4b ; iteration_count++ cmp eobd, 150 ; if (eob > 150) setg r3b ; iteration_count += 2 @@ -4672,28 +4676,13 @@ mova xm6, [cq+32* 6] mova xm7, [cq+32* 7] vinserti128 m6, m6, [cq+32*14], 1 vinserti128 m7, m7, [cq+32*15], 1 REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 - REPX {psllw x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddsw x, x }, m0, m1, m2, m3, m4, m5, m6, m7 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 - pmulhrsw m8, m10, m0 - paddw m0, m8 - pmulhrsw m8, m10, m1 - paddw m1, m8 - pmulhrsw m8, m10, m2 - paddw m2, m8 - pmulhrsw m8, m10, m3 - paddw m3, m8 - pmulhrsw m8, m10, m4 - paddw m4, m8 - pmulhrsw m8, m10, m5 - paddw m5, m8 - pmulhrsw m8, m10, m6 - paddw m6, m8 - pmulhrsw m8, m10, m7 - paddw m7, m8 + REPX {IDTX16 x, 8, 10}, 0, 1, 2, 3, 4, 5, 6, 7 REPX {pmulhrsw x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r3 lea dstq, [dstq+strideq*4] WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 @@ -4706,24 +4695,21 @@ jz .loop add cq, 32*15 lea dstq, [r5+16] jmp .loop .ret: - sub cq, 32 + sub cd, eax pxor m0, m0 - mov r0d, 4 - mov r1d, 8 - cmp cq, rax - cmovg r0d, r1d + add cd, 384 .zero_loop: mova [rax+32*0], m0 mova [rax+32*1], m0 mova [rax+32*2], m0 mova [rax+32*3], m0 add rax, 32*4 - dec r0d - jg .zero_loop + sub cd, 128 + jge .zero_loop RET cglobal inv_txfm_add_dct_dct_32x32, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd @@ -4857,11 +4843,11 @@ mova [rsp], m15 call m(idct_16x16_internal).main call m(inv_txfm_add_dct_dct_16x32).pass2_end lea tmp3q, [tmp1q-32*32] cmp tmp2q, tmp3q - jl .ret + jb .ret sub tmp2q, 32*32 sub dstq, r3 lea r2, [r2+r3+16] add dstq, 16 jmp .pass2_loop diff --git a/third_party/dav1d/src/x86/itx_ssse3.asm b/third_party/dav1d/src/x86/itx_ssse3.asm --- a/third_party/dav1d/src/x86/itx_ssse3.asm +++ b/third_party/dav1d/src/x86/itx_ssse3.asm @@ -31,13 +31,18 @@ deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 deint_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 deint_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7 -%macro COEF_PAIR 2 -pw_%1_m%2: times 4 dw %1, -%2 -pw_%2_%1: times 4 dw %2, %1 +%macro COEF_PAIR 2-3 0 ; !0 = m%1_m%2, 2 = no %2_%1 +pw_%1_m%2: times 4 dw %1, -%2 +%if %3 != 2 +pw_%2_%1: times 4 dw %2, %1 +%endif +%if %3 +pw_m%1_m%2: times 4 dw -%1, -%2 +%endif %endmacro ;adst4 pw_1321_3803: times 4 dw 1321, 3803 pw_2482_m1321: times 4 dw 2482, -1321 @@ -53,21 +58,25 @@ COEF_PAIR 3406, 2276 COEF_PAIR 401, 4076 COEF_PAIR 1931, 3612 COEF_PAIR 3166, 2598 COEF_PAIR 3920, 1189 -COEF_PAIR 3784, 1567 +COEF_PAIR 3784, 1567, 1 COEF_PAIR 995, 3973 COEF_PAIR 1751, 3703 COEF_PAIR 3513, 2106 COEF_PAIR 3857, 1380 -COEF_PAIR 4017, 799 +COEF_PAIR 4017, 799, 1 COEF_PAIR 201, 4091 COEF_PAIR 2440, 3290 COEF_PAIR 3035, 2751 COEF_PAIR 4052, 601 -COEF_PAIR 2276, 3406 +COEF_PAIR 2276, 3406, 1 +COEF_PAIR 4076, 401, 2 +COEF_PAIR 2598, 3166, 2 +COEF_PAIR 3612, 1931, 2 +COEF_PAIR 1189, 3920, 2 pd_2048: times 4 dd 2048 pw_2048: times 8 dw 2048 pw_m2048: times 8 dw -2048 pw_4096: times 8 dw 4096 @@ -75,11 +84,10 @@ pw_m16384: times 8 dw -16384 pw_1697x16: times 8 dw 1697*16 pw_1697x8: times 8 dw 1697*8 pw_2896x8: times 8 dw 2896*8 pw_3344x8: times 8 dw 3344*8 -pw_5793x4: times 8 dw 5793*4 pw_8192: times 8 dw 8192 pw_m8192: times 8 dw -8192 pw_5: times 8 dw 5 pw_201x8: times 8 dw 201*8 pw_4091x8: times 8 dw 4091*8 @@ -274,11 +282,11 @@ INV_TXFM_FN %1, %2, %3, 4x4, 6 %ifidn %1_%2, dct_identity mova m0, [o(pw_2896x8)] pmulhrsw m0, [coeffq] pmulhrsw m1, m0, [o(pw_1697x8)] - paddw m0, m1 + paddsw m0, m1 punpcklwd m0, m0 punpckhdq m1, m0, m0 punpckldq m0, m0 TAIL_CALL m(iadst_4x4_internal).end %elifidn %1_%2, identity_dct @@ -287,11 +295,11 @@ punpcklwd m0, m1, m2 punpckhwd m1, m2 punpcklwd m0, m1 punpcklqdq m0, m0 pmulhrsw m1, m0, [o(pw_1697x8)] - paddw m0, m1 + paddsw m0, m1 pmulhrsw m0, [o(pw_2896x8)] mova m1, m0 TAIL_CALL m(iadst_4x4_internal).end %elif %3 >= 0 pshuflw m0, [coeffq], q0000 @@ -437,24 +445,24 @@ mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] mova m3, [o(pw_1697x8)] pmulhrsw m2, m0, m3 pmulhrsw m3, m1 - paddw m0, m2 - paddw m1, m3 + paddsw m0, m2 + paddsw m1, m3 punpckhwd m2, m0, m1 punpcklwd m0, m1 punpckhwd m1, m0, m2 ;high: in3 ;low :in2 punpcklwd m0, m2 ;high: in1 ;low: in0 jmp tx2q .pass2: mova m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 - paddw m0, m2 - paddw m1, m3 + paddsw m0, m2 + paddsw m1, m3 jmp m(iadst_4x4_internal).end %macro IWHT4_1D_PACKED 0 punpckhqdq m3, m0, m1 ;low: in1 high: in3 punpcklqdq m0, m1 ;low: in0 high: in2 @@ -609,11 +617,11 @@ punpcklwd m1, [coeffq+16*3] mova m2, [o(pw_2896x8)] punpckldq m0, m1 pmulhrsw m0, m2 pmulhrsw m1, m0, [o(pw_1697x8)] - paddw m0, m1 + paddsw m0, m1 pmulhrsw m0, m2 pmulhrsw m0, [o(pw_2048)] punpcklqdq m0, m0 mova m1, m0 mova m2, m0 @@ -832,14 +840,14 @@ mova m7, [o(pw_1697x8)] pmulhrsw m4, m7, m0 pmulhrsw m5, m7, m1 pmulhrsw m6, m7, m2 pmulhrsw m7, m3 - paddw m0, m4 - paddw m1, m5 - paddw m2, m6 - paddw m3, m7 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 jmp m(iadst_4x8_internal).pass1_end .pass2: mova m4, [o(pw_4096)] jmp m(iadst_4x8_internal).end2 @@ -879,11 +887,11 @@ %ifidn %1_%2, dct_identity mova m0, [o(pw_2896x8)] pmulhrsw m1, m0, [coeffq] pmulhrsw m1, m0 pmulhrsw m0, m1, [o(pw_1697x8)] - paddw m1, m0 + paddsw m1, m0 pmulhrsw m1, [o(pw_2048)] punpcklwd m1, m1 punpckhdq m2, m1, m1 punpckldq m1, m1 punpckhdq m3, m2, m2 @@ -902,11 +910,11 @@ punpcklwd m0, m4 punpcklwd m2, m5 punpcklqdq m0, m2 mova m4, [o(pw_2896x8)] pmulhrsw m0, m4 - paddw m0, m0 + paddsw m0, m0 pmulhrsw m0, m4 pmulhrsw m0, [o(pw_2048)] mova m1, m0 mova m2, m0 mova m3, m0 @@ -1006,12 +1014,12 @@ punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 pxor m5, m5 - psubw m3, m5, m1 - psubw m5, m4 + psubsw m3, m5, m1 + psubsw m5, m4 punpckhdq m4, m5, m3 punpckldq m5, m3 punpckhdq m3, m0, m2 punpckldq m0, m2 punpckhwd m1, m0, m5 ;in1 @@ -1125,12 +1133,12 @@ punpcklwd m3, m2 punpckhwd m2, m1, m0 punpcklwd m1, m0 pxor m0, m0 - psubw m4, m0, m2 - psubw m0, m5 + psubsw m4, m0, m2 + psubsw m0, m5 punpckhdq m2, m0, m4 punpckldq m0, m4 punpckhdq m4, m3, m1 punpckldq m3, m1 punpckhwd m1, m0, m3 ;in1 @@ -1158,14 +1166,14 @@ mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] - paddw m0, m0 - paddw m1, m1 - paddw m2, m2 - paddw m3, m3 + paddsw m0, m0 + paddsw m1, m1 + paddsw m2, m2 + paddsw m3, m3 punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 @@ -1183,14 +1191,14 @@ mova m7, [o(pw_1697x8)] pmulhrsw m4, m7, m0 pmulhrsw m5, m7, m1 pmulhrsw m6, m7, m2 pmulhrsw m7, m3 - paddw m0, m4 - paddw m1, m5 - paddw m2, m6 - paddw m3, m7 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 jmp m(iadst_8x4_internal).end %macro INV_TXFM_8X8_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 8x8, 8, 16*4 %ifidn %1_%2, dct_identity @@ -1639,14 +1647,14 @@ mova m4, [o(pw_2048)] pmulhrsw m0, m2 pmulhrsw m1, m2 pmulhrsw m2, m3, m0 pmulhrsw m3, m1 - paddw m0, m0 - paddw m1, m1 - paddw m0, m2 - paddw m1, m3 + paddsw m0, m0 + paddsw m1, m1 + paddsw m0, m2 + paddsw m1, m3 pmulhrsw m0, m4 pmulhrsw m4, m1 punpckhwd m2, m0, m0 punpcklwd m0, m0 punpckhwd m6, m4, m4 @@ -1664,22 +1672,21 @@ %elifidn %1_%2, identity_dct movd m0, [coeffq+32*0] punpcklwd m0, [coeffq+32*1] movd m1, [coeffq+32*2] punpcklwd m1, [coeffq+32*3] - mova m3, [o(pw_16384)] punpckldq m0, m1 pmulhrsw m1, m0, [o(pw_1697x8)] - mova m4, [o(pw_2896x8)] - paddw m0, m1 - pmulhrsw m0, m3 - psrlw m3, 3 ; pw_2048 - pmulhrsw m0, m4 - pmulhrsw m0, m3 + pcmpeqw m2, m2 + pcmpeqw m2, m0 + pxor m0, m2 + pavgw m0, m1 + pmulhrsw m0, [o(pw_2896x8)] + pmulhrsw m0, [o(pw_2048)] punpcklqdq m0, m0 - pxor m7, m7 - REPX {mova [coeffq+32*x], m7}, 0, 1, 2, 3 + pxor m1, m1 + REPX {mova [coeffq+32*x], m1}, 0, 1, 2, 3 %elifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklwd m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 @@ -1884,37 +1891,79 @@ INV_TXFM_4X16_FN identity, dct, 3 INV_TXFM_4X16_FN identity, adst INV_TXFM_4X16_FN identity, flipadst INV_TXFM_4X16_FN identity, identity -%macro IDTX16 3 ; src/dst, tmp, pw_1697x16 - pmulhrsw m%2, m%3, m%1 - paddw m%1, m%1 - paddw m%1, m%2 +%macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16394] + pmulhrsw m%2, m%3, m%1 +%if %0 == 4 ; if downshifting by 1 + pmulhrsw m%2, m%4 +%else + paddsw m%1, m%1 +%endif + paddsw m%1, m%2 %endmacro cglobal iidentity_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 - lea r3, [o(m(iidentity_4x8_internal).pass1)] - jmp m(idct_4x16_internal).pass1 - + mova m0, [coeffq+16*1] + mova m6, [o(pw_1697x8)] + mova m1, [coeffq+16*3] + mova m2, [coeffq+16*5] + mova m3, [coeffq+16*7] + pcmpeqw m7, m7 + mov r3, tx2q + lea tx2q, [o(.pass1_2)] +.pass1: + pmulhrsw m4, m6, m0 + pmulhrsw m5, m6, m1 + pavgw m4, m0 + pcmpeqw m0, m7 + pavgw m5, m1 + pcmpeqw m1, m7 + pandn m0, m4 + pmulhrsw m4, m6, m2 + pandn m1, m5 + pmulhrsw m5, m6, m3 + pavgw m4, m2 + pcmpeqw m2, m7 + pavgw m5, m3 + pcmpeqw m3, m7 + pandn m2, m4 + pandn m3, m5 + jmp m(iadst_4x8_internal).pass1_end +.pass1_2: + mova [coeffq+16*1], m0 + mova [coeffq+16*3], m1 + mova [coeffq+16*5], m2 + mova [coeffq+16*7], m3 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*2] + mova m2, [coeffq+16*4] + mova m3, [coeffq+16*6] + lea tx2q, [o(.pass1_end)] + jmp .pass1 +.pass1_end: + mova m4, [coeffq+16*1] + mova m5, [coeffq+16*3] + mova m6, [coeffq+16*5] + jmp r3 .pass2: mova m7, [o(pw_1697x16)] mova [coeffq+16*6], m6 REPX {IDTX16 x, 6, 7}, 0, 1, 2, 3, 4, 5 mova m6, [coeffq+16*7] IDTX16 6, 7, 7 mova [coeffq+16*7], m6 mova m6, [coeffq+16*6] pmulhrsw m7, m6, [o(pw_1697x16)] - paddw m6, m6 - paddw m6, m7 - + paddsw m6, m6 + paddsw m6, m7 mova m7, [o(pw_2048)] REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 pmulhrsw m7, [coeffq+16*7] mova [coeffq+16*4], m4 - jmp m(iadst_4x16_internal).end2 + jmp m(iadst_4x16_internal).end2 %macro INV_TXFM_16X4_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 16x4, 8 %if %3 >= 0 @@ -1923,11 +1972,11 @@ pmulhrsw m3, [coeffq] mova m0, [o(pw_16384)] pmulhrsw m3, m0 psrlw m0, 3 ; pw_2048 pmulhrsw m1, m3, [o(pw_1697x8)] - paddw m3, m1 + paddsw m3, m1 pmulhrsw m3, m0 punpcklwd m3, m3 pshufd m0, m3, q0000 pshufd m1, m3, q1111 pshufd m2, m3, q2222 @@ -1955,11 +2004,11 @@ punpcklwd m1, m2 punpcklwd m1, m3 punpcklqdq m0, m1 pmulhrsw m1, m4, m0 pmulhrsw m1, m5 - paddw m0, m1 + paddsw m0, m1 pmulhrsw m0, m6 pmulhrsw m0, m7 .end: pxor m3, m3 mova [coeffq+16*0], m3 @@ -1975,11 +2024,11 @@ RET %else movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] %ifidn %2, dct - movd m2, [o(pw_16384)] + movd m2, [o(pw_16384)] mov [coeffq], eobd mov r2d, 2 lea tx2q, [o(m(inv_txfm_add_dct_dct_16x4).end)] .dconly: pmulhrsw m0, m2 @@ -2072,19 +2121,17 @@ ITX_MUL2X_PACK %4, %6, %7, 3166, 2598, 1 ;low: t9a high: t14a ITX_MUL2X_PACK %2, %6, %7, 1931, 3612, 1 ;low: t10a high: t13a ITX_MUL2X_PACK %5, %6, %7, 3920, 1189, 1 ;low: t11a high: t12a psubsw m%6, m%1, m%4 ;low: t9 high: t14 paddsw m%1, m%4 ;low: t8 high: t15 - psubsw m%3, m%5, m%2 ;low: t10 high: t13 + psubsw m%4, m%5, m%2 ;low: t10 high: t13 paddsw m%5, m%2 ;low: t11 high: t12 mova m%2, [o(deint_shuf2)] pshufb m%6, m%2 - pshufb m%3, [o(deint_shuf1)] - pxor m%4, m%4 - psubw m%4, m%3 ;packed -t10 -t13 + pshufb m%4, m%2 ITX_MUL2X_PACK %6, %3, %7, 1567, 3784, 1 ;low: t9a high: t14a - ITX_MUL2X_PACK %4, %3, %7, 3784, 1567 ;low: t10a high: t13a + ITX_MUL2X_PACK %4, %3, %7, m3784, 1567, 1 ;low: t10a high: t13a psubsw m%3, m%1, m%5 ;low: t11a high: t12a paddsw m%1, m%5 ;low: t8a high: t15a psubsw m%5, m%6, m%4 ;low: t10 high: t13 paddsw m%6, m%4 ;low: t9 high: t14 pshufb m%3, m%2 @@ -2432,13 +2479,13 @@ pmulhrsw m3, m6, m0 pmulhrsw m5, m6, m2 pmulhrsw m4, m7 pmulhrsw m3, m7 pmulhrsw m5, m7 - paddw m1, m4 - paddw m0, m3 - paddw m5, m2 + paddsw m1, m4 + paddsw m0, m3 + paddsw m5, m2 mova m2, [coeffq+16*2] mova m3, [coeffq+16*3] mova m4, [coeffq+16*4] mova [coeffq+16*6], m1 mova [coeffq+16*5], m0 @@ -2447,21 +2494,21 @@ pmulhrsw m1, m6, m3 pmulhrsw m5, m6, m4 pmulhrsw m0, m7 pmulhrsw m1, m7 pmulhrsw m5, m7 - paddw m2, m0 - paddw m3, m1 - paddw m4, m5 + paddsw m2, m0 + paddsw m3, m1 + paddsw m4, m5 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] pmulhrsw m5, m6, m0 pmulhrsw m6, m1 pmulhrsw m5, m7 pmulhrsw m6, m7 - paddw m0, m5 - paddw m1, m6 + paddsw m0, m5 + paddsw m1, m6 mova m6, [coeffq+16*6] mova m5, [coeffq+16*5] punpckhwd m7, m0, m2 ;packed out1, out5 punpcklwd m0, m2 ;packed out0, out4 punpckhwd m2, m1, m3 ;packed out3, out7 @@ -2519,12 +2566,12 @@ mova [coeffq], m2 pmulhrsw m7, m0 pmulhrsw m7, m1 psrlw m1, 3 ; pw_2048 pmulhrsw m0, m7, [o(pw_1697x16)] - paddw m7, m7 - paddw m7, m0 + paddsw m7, m7 + paddsw m7, m0 pmulhrsw m7, m1 punpcklwd m0, m7, m7 punpckhwd m7, m7 pshufd m3, m0, q3333 pshufd m2, m0, q2222 @@ -2856,11 +2903,11 @@ punpckldq m1, m2 punpcklqdq m0, m1 pmulhrsw m0, m4 pmulhrsw m1, m5, m0 pmulhrsw m1, m6 - paddw m0, m1 + paddsw m0, m1 pmulhrsw m0, m4 pmulhrsw m0, m7 .end: pxor m1, m1 REPX {mova [coeffq+16*x], m1}, 0, 1, 2, 3, 4, 5, 6, 7 @@ -2931,36 +2978,34 @@ mova [rsp+gprsize*2+32*5], m4 mova m2, [rsp+gprsize*2+16*2] mova [rsp+gprsize*2+16*2], m7 ITX_MULSUB_2W 3, 5, 7, 4, 6, 1931, 3612 ;t10a, t13a ITX_MULSUB_2W 2, 1, 7, 4, 6, 3920, 1189 ;t11a, t12a - pxor m4, m4 - psubsw m7, m2, m3 ;t10 + psubsw m4, m2, m3 ;t10 paddsw m2, m3 ;t11 psubsw m3, m1, m5 ;t13 paddsw m1, m5 ;t12 - psubw m4, m7 - ITX_MULSUB_2W 4, 3, 7, 5, 6, 1567, 3784 ;t10a, t13a + ITX_MULSUB_2W 3, 4, 7, 5, 6, m3784, 1567 ;t10a, t13a mova m7, [rsp+gprsize*2+32*5] psubsw m6, m0, m2 ;t11a paddsw m0, m2 ;t8a - paddsw m2, m7, m4 ;t9 - psubsw m7, m4 ;t10 + paddsw m2, m7, m3 ;t9 + psubsw m7, m3 ;t10 mova m5, [rsp+gprsize*2+16*0] - psubsw m4, m5, m0 ;out8 + psubsw m3, m5, m0 ;out8 paddsw m0, m5 ;out7 mova [rsp+gprsize*2+32*5], m0 mova m5, [rsp+gprsize*2+16*9] psubsw m0, m5, m2 ;out9 paddsw m2, m5 ;out6 mova [rsp+gprsize*2+16*0], m0 mova [rsp+gprsize*2+16*9], m2 mova m0, [rsp+gprsize*2+16*1] mova m2, [rsp+gprsize*2+16*2] - mova [rsp+gprsize*2+16*1], m4 - psubsw m5, m0, m3 ;t13 - paddsw m0, m3 ;t14 + mova [rsp+gprsize*2+16*1], m3 + psubsw m5, m0, m4 ;t13 + paddsw m0, m4 ;t14 mova m3, [o(pd_2048)] psubsw m4, m2, m1 ;t12a paddsw m1, m2 ;t15a mova [rsp+gprsize*2+16*2], m1 ITX_MULSUB_2W 5, 7, 1, 2, 3, 2896, 2896 ;t10a, t13a @@ -3357,39 +3402,39 @@ mova m3, [o(pw_16384)] sub coeffq, 8*16 REPX {pmulhrsw x, m0}, m4, m5, m6, m7 pmulhrsw m1, m2, m4 pmulhrsw m1, m3 - paddw m1, m4 ; 1 + paddsw m1, m4 ; 1 pmulhrsw m4, m2, m5 pmulhrsw m4, m3 - paddw m4, m5 ; 3 + paddsw m4, m5 ; 3 pmulhrsw m5, m2, m6 pmulhrsw m5, m3 - paddw m5, m6 ; 5 + paddsw m5, m6 ; 5 pmulhrsw m6, m2, m7 pmulhrsw m6, m3 - paddw m7, m6 ; 7 + paddsw m7, m6 ; 7 pmulhrsw m6, m0, [coeffq+16*6] mova [rsp+gprsize+16*0], m4 pmulhrsw m4, m2, m6 pmulhrsw m4, m3 - paddw m6, m4 ; 6 + paddsw m6, m4 ; 6 pmulhrsw m4, m0, [coeffq+16*4] mova [rsp+gprsize+16*1], m6 pmulhrsw m6, m2, m4 pmulhrsw m6, m3 - paddw m4, m6 ; 4 + paddsw m4, m6 ; 4 pmulhrsw m6, m0, [coeffq+16*2] pmulhrsw m0, [coeffq+16*0] pmulhrsw m2, m6 pmulhrsw m2, m3 - paddw m2, m6 ; 2 + paddsw m2, m6 ; 2 pmulhrsw m6, m0, [o(pw_1697x16)] pmulhrsw m6, m3 mova m3, [rsp+gprsize+16*0] - paddw m0, m6 + paddsw m0, m6 jmp m(idct_8x8_internal).pass1_end3 .pass1_end: mova [coeffq+16*1], m4 mova [coeffq+16*3], m5 @@ -3433,25 +3478,27 @@ %elifidn %1_%2, dct_identity mova m3, [o(pw_2896x8)] pmulhrsw m2, m3, [coeffq+16*0] pmulhrsw m3, [coeffq+16*1] mova m0, [o(pw_8192)] - mova m1, [o(pw_5793x4)] + mova m1, [o(pw_1697x16)] pshuflw m4, [o(deint_shuf)], q0000 ;pb_0_1 punpcklwd m4, m4 pcmpeqb m5, m5 pxor m6, m6 mova [coeffq+16*0], m6 mova [coeffq+16*1], m6 paddb m5, m5 ;pb_m2 pmulhrsw m2, m0 pmulhrsw m3, m0 psrlw m0, 2 ;pw_2048 - psllw m2, 2 - psllw m3, 2 - pmulhrsw m2, m1 - pmulhrsw m3, m1 + pmulhrsw m7, m1, m2 + pmulhrsw m1, m3 + paddsw m2, m2 + paddsw m3, m3 + paddsw m2, m7 + paddsw m3, m1 pmulhrsw m2, m0 pmulhrsw m3, m0 mov r3d, 8 .loop: mova m1, [dstq] @@ -3475,13 +3522,12 @@ dec r3d jg .loop RET %elifidn %1_%2, identity_dct mova m4, [o(pw_1697x16)] - mova m5, [o(pw_8192)] - mova m6, [o(pw_2896x8)] - psrlw m7, m5, 2 ;pw_2048 + mova m5, [o(pw_2896x8)] + mova m6, [o(pw_2048)] xor eobd, eobd lea tx2q, [o(m(inv_txfm_add_identity_dct_16x16).end)] lea r3, [dstq+8] mov [rsp+16*0], r3 .main: @@ -3501,15 +3547,14 @@ mov [coeffq-32*2], eobd mov [coeffq-32*1], eobd punpckldq m1, m2 punpcklqdq m0, m1 pmulhrsw m1, m4, m0 - paddw m0, m0 - paddw m0, m1 + psraw m1, 1 + pavgw m0, m1 pmulhrsw m0, m5 pmulhrsw m0, m6 - pmulhrsw m0, m7 mov [coeffq+32*0], eobd mov [coeffq+32*1], eobd mov [coeffq+32*2], eobd mov [coeffq+32*3], eobd mov r3d, 4 @@ -3812,34 +3857,39 @@ lea tx2q, [o(m(idct_8x16_internal).end1)] lea dstq, [dstq+strideq*2] jmp m(iflipadst_8x8_internal).end +%macro IDTX16B 3 ; src/dst, tmp, pw_1697x16 + pmulhrsw m%2, m%3, m%1 + psraw m%2, 1 + pavgw m%1, m%2 +%endmacro + INV_TXFM_16X16_FN identity, dct, 15 INV_TXFM_16X16_FN identity, identity cglobal iidentity_16x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 add coeffq, 16*17 mov r3, tx2q lea tx2q, [o(m(iidentity_16x16_internal).pass1_end)] .pass1: - mova m7, [o(pw_1697x16)] - mova m6, [coeffq+32*7] + mova m6, [o(pw_1697x16)] + mova m7, [coeffq+32*6] mova m0, [coeffq+32*0] mova m1, [coeffq+32*1] mova m2, [coeffq+32*2] mova m3, [coeffq+32*3] mova m4, [coeffq+32*4] - REPX {IDTX16 x, 5, 7}, 6, 0, 1, 2, 3, 4 + REPX {IDTX16B x, 5, 6}, 7, 0, 1, 2, 3, 4 mova m5, [coeffq+32*5] - mova [rsp+gprsize+16*0], m6 - IDTX16 5, 6, 7 - mova m6, [coeffq+32*6] - IDTX16 6, 7, 7 - mova m7, [o(pw_8192)] - jmp m(idct_8x8_internal).pass1_end1 + mova [rsp+gprsize+16*1], m7 + IDTX16B 5, 7, 6 + mova m7, [coeffq+32*7] + IDTX16B 7, 6, 6 + jmp m(idct_8x8_internal).pass1_end3 .pass1_end: SAVE_8ROWS coeffq, 32 sub coeffq, 16 lea tx2q, [o(m(iidentity_16x16_internal).pass1_end1)] @@ -4092,15 +4142,13 @@ mova m1, [rsp+gprsize*2+16*22] ;in7 pmulhrsw m2, m1, [o(pw_3857x8)] ;t28,t29 pmulhrsw m1, [o(pw_m1380x8)] ;t18,t19 mova [rsp+gprsize*2+16*22], m1 ;t19 mova [rsp+gprsize*2+16*31], m2 ;t28 - pxor m0, m0 - psubw m0, m1 - ITX_MULSUB_2W 0, 2, 1, 3, 7, 799, 4017 ;t18a, t29a - mova [rsp+gprsize*2+16*21], m0 ;t18a - mova [rsp+gprsize*2+16*32], m2 ;t29a + ITX_MULSUB_2W 2, 1, 0, 3, 7, m4017, 799 ;t18a, t29a + mova [rsp+gprsize*2+16*21], m2 ;t18a + mova [rsp+gprsize*2+16*32], m1 ;t29a mova m0, [rsp+gprsize*2+16*23] ;in5 pmulhrsw m3, m0, [o(pw_3973x8)] ;t26, t27 pmulhrsw m0, [o(pw_995x8)] ;t20, t21 mova [rsp+gprsize*2+16*23], m0 ;t20 mova [rsp+gprsize*2+16*30], m3 ;t27 @@ -4140,17 +4188,15 @@ pmulhrsw m1, [o(pw_m1380x8)] psubsw m4, m1, m0 ;t18 paddsw m0, m1 ;t19 psubsw m5, m2, m3 ;t29 paddsw m3, m2 ;t28 - pxor m2, m2 - psubw m2, m4 - ITX_MULSUB_2W 2, 5, 1, 4, 7, 799, 4017 ;t18a, t29a - mova [rsp+gprsize*2+16*21], m2 ;t18a + ITX_MULSUB_2W 5, 4, 1, 2, 7, m4017, 799 ;t18a, t29a + mova [rsp+gprsize*2+16*21], m5 ;t18a mova [rsp+gprsize*2+16*22], m0 ;t19 mova [rsp+gprsize*2+16*31], m3 ;t28 - mova [rsp+gprsize*2+16*32], m5 ;t29a + mova [rsp+gprsize*2+16*32], m4 ;t29a mova m0, [rsp+gprsize*2+16*23] ;in5 mova m1, [rsp+gprsize*2+16*24] ;in11 pmulhrsw m3, m0, [o(pw_3973x8)] pmulhrsw m0, [o(pw_995x8)] pmulhrsw m2, m1, [o(pw_3513x8)] @@ -4198,17 +4244,15 @@ ITX_MULSUB_2W 2, 1, 4, 5, 7, 3857, 1380 ;t19a, t28a psubsw m4, m2, m0 ;t18 paddsw m0, m2 ;t19 psubsw m5, m1, m3 ;t29 paddsw m3, m1 ;t28 - pxor m2, m2 - psubw m2, m4 ;-t18 - ITX_MULSUB_2W 2, 5, 1, 4, 7, 799, 4017 ;t18a, t29a - mova [rsp+gprsize*2+16*21], m2 ;t18a + ITX_MULSUB_2W 5, 4, 1, 2, 7, m4017, 799 ;t18a, t29a + mova [rsp+gprsize*2+16*21], m5 ;t18a mova [rsp+gprsize*2+16*22], m0 ;t19 mova [rsp+gprsize*2+16*31], m3 ;t28 - mova [rsp+gprsize*2+16*32], m5 ;t29a + mova [rsp+gprsize*2+16*32], m4 ;t29a mova m0, [rsp+gprsize*2+16*23] ;in5 mova m1, [rsp+gprsize*2+16*24] ;in11 mova m2, [rsp+gprsize*2+16*29] ;in21 mova m3, [rsp+gprsize*2+16*30] ;in27 ITX_MULSUB_2W 0, 3, 4, 5, 7, 995, 3973 ;t20a, t27a @@ -4232,38 +4276,33 @@ .main2: psubsw m4, m2, m0 ;t22 paddsw m0, m2 ;t23 psubsw m5, m1, m3 ;t25 paddsw m3, m1 ;t24 - pxor m6, m6 - psubw m2, m6, m4 - ITX_MULSUB_2W 2, 5, 1, 4, 7, 3406, 2276 ;t22a, t25a - - mova m4, [rsp+gprsize*2+16*24] ;t21a - psubsw m1, m2, m4 ;t21 - paddsw m2, m4 ;t22 - psubw m4, m6, m1 ;-t21 - mova [rsp+gprsize*2+16*25], m2 ;t22 - mova m1, [rsp+gprsize*2+16*29] ;t26a - psubsw m2, m5, m1 ;t26 - paddsw m5, m1 ;t25 - mova [rsp+gprsize*2+16*28], m5 ;t25 - ITX_MULSUB_2W 4, 2, 1, 5, 7, 1567, 3784 ;t21a, t26a - mova [rsp+gprsize*2+16*24], m4 ;t21a - mova [rsp+gprsize*2+16*29], m2 ;t26a + ITX_MULSUB_2W 5, 4, 1, 2, 7, m2276, 3406 ;t22a, t25a + mova m2, [rsp+gprsize*2+16*24] ;t21a + psubsw m1, m5, m2 ;t21 + paddsw m5, m2 ;t22 + mova [rsp+gprsize*2+16*25], m5 ;t22 + mova m2, [rsp+gprsize*2+16*29] ;t26a + psubsw m5, m4, m2 ;t26 + paddsw m4, m2 ;t25 + mova [rsp+gprsize*2+16*28], m4 ;t25 + ITX_MULSUB_2W 5, 1, 2, 4, 7, m3784, 1567 ;t21a, t26a + mova [rsp+gprsize*2+16*24], m5 ;t21a + mova [rsp+gprsize*2+16*29], m1 ;t26a mova m1, [rsp+gprsize*2+16*23] ;t20 mova m5, [rsp+gprsize*2+16*30] ;t27 psubsw m2, m0, m1 ;t20a paddsw m0, m1 ;t23a - psubsw m4, m3, m5 ;t27a + psubsw m6, m3, m5 ;t27a paddsw m3, m5 ;t24a - psubw m6, m2 ;-t20a - ITX_MULSUB_2W 6, 4, 1, 5, 7, 1567, 3784 ;t20, t27 + ITX_MULSUB_2W 6, 2, 1, 5, 7, m3784, 1567 ;t20, t27 mova [rsp+gprsize*2+16*26], m0 ;t23a mova [rsp+gprsize*2+16*27], m3 ;t24a - mova [rsp+gprsize*2+16*30], m4 ;t27 + mova [rsp+gprsize*2+16*30], m2 ;t27 mova m0, [rsp+gprsize*2+16*20] ;t17a mova m1, [rsp+gprsize*2+16*21] ;t18a mova m2, [rsp+gprsize*2+16*32] ;t29a mova m3, [rsp+gprsize*2+16*33] ;t30a @@ -4442,11 +4481,11 @@ mova [dstq+16*0], m1 mova [dstq+16*1], m3 add dstq, strideq dec r3d jg .loop - jmp tx2q + jmp tx2q .end: RET @@ -4510,11 +4549,11 @@ lea tx2q, [o(m(idct_32x8_internal).end3)] jmp m(idct_8x8_internal).pass1_end1 .end3: mov dstq, r3 - lea r3, [r3+8] + add r3, 8 lea tx2q, [o(m(idct_32x8_internal).end4)] jmp m(idct_8x8_internal).pass2_main .end4: LOAD_8ROWS rsp+gprsize+16*19, 16 @@ -4523,11 +4562,11 @@ lea tx2q, [o(m(idct_32x8_internal).end5)] jmp m(idct_8x8_internal).pass1_end1 .end5: mov dstq, r3 - lea r3, [r3+8] + add r3, 8 lea tx2q, [o(m(idct_32x8_internal).end6)] jmp m(idct_8x8_internal).pass2_main .end6: LOAD_8ROWS rsp+gprsize+16*27, 16 @@ -4546,47 +4585,42 @@ cglobal inv_txfm_add_identity_identity_8x32, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 mov r5d, 4 mov tx2d, 2 - cmp eobd, 106 - cmovg tx2d, r5d + cmp eobd, 107 + cmovns tx2d, r5d mov r3d, tx2d %if ARCH_X86_32 LEA r5, $$ %endif lea tx2q, [o(m(idct_32x8_internal).end8)] - .loop: LOAD_8ROWS coeffq+16*0, 64 - paddw m6, [o(pw_5)] + paddsw m6, [o(pw_5)] mova [rsp+16*1], m6 mova m6, [o(pw_5)] - REPX {paddw x, m6}, m0, m1, m2, m3, m4, m5, m7 - + REPX {paddsw x, m6}, m0, m1, m2, m3, m4, m5, m7 call m(idct_8x8_internal).pass1_end3 - REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 - + REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 mova [rsp+16*2], m5 mova [rsp+16*1], m6 mova [rsp+16*0], m7 call m(idct_8x8_internal).end3 lea dstq, [dstq+strideq*2] - pxor m7, m7 REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 - add coeffq, 16 dec r3d jg .loop RET cglobal inv_txfm_add_identity_identity_32x8, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 mov r5d, 4 mov tx2d, 2 - cmp eobd, 106 - cmovg tx2d, r5d + cmp eobd, 107 + cmovns tx2d, r5d mov r3d, tx2d %if ARCH_X86_32 LEA r5, $$ %endif @@ -4685,11 +4719,11 @@ mova [rsp+gprsize+16*19], m1 ;in1 mova [rsp+gprsize+16*26], m3 ;in3 mova [rsp+gprsize+16*23], m5 ;in5 mova [rsp+gprsize+16*22], m7 ;in7 - cmp eobd, 150 + cmp eobd, 150 jg .full mova m1, m4 ;in4 mova m2, [coeffq+16*1 ] ;in8 mova m3, [coeffq+16*5 ] ;in12 @@ -5000,138 +5034,123 @@ cglobal inv_txfm_add_identity_identity_16x32, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 %undef cmp - mov r4, 1 - mov r5, 2 + mov r4d, eobd cmp eobd, 43 ;if (eob > 43) - cmovg r4, r5 ; iteration_count++ - inc r5 - cmp eobd, 150 ;if (eob > 150) - cmovg r4, r5 ; iteration_count++ - inc r5 - cmp eobd, 278 ;if (eob > 278) - cmovg r4, r5 ; iteration_count++ + sbb r3d, r3d ; iteration_count++ + cmp r4d, 150 ;if (eob > 150) + sbb r3d, 0 ; iteration_count++ + cmp r4d, 278 ;if (eob > 278) + sbb r3d, -4 ; iteration_count++ %if ARCH_X86_32 LEA r5, $$ %endif - lea r3, [dstq+8] - mov [rsp+16*3], r3 - mov r3, r4 - mov [rsp+gprsize+16*3], r4 + lea r4, [dstq+8] + mov [rsp+16*3], r4 + mov [rsp+gprsize+16*3], r3d mov [rsp+gprsize*2+16*3], coeffq .loop: LOAD_8ROWS coeffq, 64, 1 - REPX {psllw x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 mova [rsp+16*1], m6 + pxor m6, m6 + REPX {mova [coeffq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 lea tx2q, [o(m(idct_32x16_internal).end)] call m(idct_8x8_internal).pass1_end3 - pmulhrsw m7, [o(pw_5793x4)] - paddw m7, [o(pw_5)] - psraw m7, 3 + mova [rsp+16*0], m2 + mova [rsp+16*1], m3 + mova [rsp+16*2], m4 + mova m3, [o(pw_1697x16)] + mova m4, [o(pw_16384)] + REPX {IDTX16 x, 2, 3, 4}, 5, 6, 7, 0, 1 + mova m2, [o(pw_8192)] + REPX {pmulhrsw x, m2}, m5, m6, m7, m0, m1 + mova m2, [rsp+16*0] mova [rsp+16*0], m7 - mova m7, [o(pw_5793x4)] - REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 - mova m7, [o(pw_5)] - REPX {paddw x, m7}, m0, m1, m2, m3, m4, m5, m6 - REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6 + IDTX16 2, 7, 3, 4 + mova m7, [rsp+16*2] mova [rsp+16*2], m5 + IDTX16 7, 5, 3, 4 + mova m5, [rsp+16*1] mova [rsp+16*1], m6 + pmulhrsw m3, m5 + pmulhrsw m3, m4 + psrlw m4, 1 ; pw_8192 + paddsw m3, m5 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + pmulhrsw m4, m7 call m(idct_8x8_internal).end3 lea dstq, [dstq+strideq*2] - - pxor m7, m7 - REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 - add coeffq, 16 - dec r3 + dec r3d jg .loop - mov coeffq, [rsp+gprsize*2+16*3] add coeffq, 64*8 - mov r3, [rsp+gprsize+16*3] + mov r3d, [rsp+gprsize+16*3] xor dstq, dstq mov [rsp+gprsize+16*3], dstq mov dstq, [rsp+16*3] - test r3, r3 + test r3d, r3d jnz .loop - RET cglobal inv_txfm_add_identity_identity_32x16, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 %undef cmp - mov r4, 12 ;0100b - mov r5, 136 ;1000 1000b - cmp eobd, 43 ;if (eob > 43) - cmovg r4, r5 ; iteration_count+2 - mov r5, 34952 ;1000 1000 1000 1000b - cmp eobd, 150 ;if (eob > 150) - cmovg r4, r5 ; iteration_count += 4 + mov r4d, 12 ;0100b + mov r5d, 136 ;1000 1000b + cmp eobd, 44 ;if (eob > 43) + cmovns r4d, r5d ; iteration_count+2 + cmp eobd, 151 ;if (eob > 150) + mov r3d, 34952 ;1000 1000 1000 1000b + cmovs r3d, r4d ; iteration_count += 4 %if ARCH_X86_32 LEA r5, $$ %endif - lea r3, [dstq+8] - mov [rsp+16*3], r3 - mov r3, r4 + lea r4, [dstq+8] + mov [rsp+16*3], r4 .loop: LOAD_8ROWS coeffq, 32, 1 - REPX {psllw x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddsw x, x}, m0, m1, m2, m3, m4, m5, m6, m7 mova [rsp+16*1], m6 lea tx2q, [o(m(idct_32x16_internal).end)] call m(idct_8x8_internal).pass1_end3 + mova [rsp+16*1], m5 + mova [rsp+16*2], m6 + mova m6, [o(pw_1697x16)] + REPX {IDTX16 x, 5, 6}, 7, 0, 1, 2, 3, 4 + pmulhrsw m7, [o(pw_2048)] + mova m5, [rsp+16*1] + mova [rsp+16*0], m7 + IDTX16 5, 7, 6 + mova m7, [rsp+16*2] + IDTX16 7, 6, 6 + mova m6, [o(pw_2048)] + REPX {pmulhrsw x, m6}, m0, m1, m2, m3, m4, m5, m7 mova [rsp+16*2], m5 - mova [rsp+16*1], m6 - mova m5, [o(pw_1697x8)] - pmulhrsw m6, m5, m7 - paddw m7, m6 - pmulhrsw m6, m5, m0 - paddw m0, m6 - pmulhrsw m6, m5, m1 - paddw m1, m6 - pmulhrsw m6, m5, m2 - paddw m2, m6 - pmulhrsw m6, m5, m3 - paddw m3, m6 - pmulhrsw m6, m5, m4 - pmulhrsw m7, [o(pw_2048)] - paddw m4, m6 - mova m6, [rsp+16*1] - mova [rsp+16*0], m7 - pmulhrsw m7, m5, m6 - paddw m6, m7 - mova m7, [rsp+16*2] - pmulhrsw m5, m7 - paddw m5, m7 - mova m7, [o(pw_2048)] - REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 - mova [rsp+16*2], m5 - mova [rsp+16*1], m6 + mova [rsp+16*1], m7 call m(idct_8x8_internal).end3 lea dstq, [dstq+strideq*2] - pxor m7, m7 REPX {mova [coeffq+32*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 .loop_end: add coeffq, 16 - shr r3, 2 - test r3, r3 + shr r3d, 2 jz .ret - test r3, 2 + test r3d, 2 jnz .loop - mov r4, r3 - and r4, 1 - shl r4, 3 - add coeffq, r4 - add coeffq, 32*7 + mov r4d, r3d + and r4d, 1 + lea coeffq, [coeffq+r4*8+32*7] mov dstq, [rsp+16*3] lea r4, [dstq+8] mov [rsp+16*3], r4 jmp .loop @@ -5160,21 +5179,20 @@ cglobal idct_32x32_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp - mov r5, 4 - mov r4, 2 + mov r4d, 2 sub eobd, 136 - cmovge r4, r5 + mov [rsp+gprsize*1+16*35], eobd + mov r3d, 4 + cmovs r3d, r4d %if ARCH_X86_32 LEA r5, $$ %endif - mov [rsp+gprsize*1+16*35], eobd - mov r3, r4 mov [rsp+gprsize*2+16*35], coeffq .pass1_loop: LOAD_8ROWS coeffq+64*1, 64*2 mova [rsp+gprsize+16*19], m0 ;in1 @@ -5266,21 +5284,21 @@ .pass1_end4: SAVE_8ROWS coeffq+64*24, 64 add coeffq, 16 - dec r3 + dec r3d jg .pass1_loop .pass2: mov coeffq, [rsp+gprsize*2+16*35] - mov r3, 4 + mov r3d, 4 lea tx2q, [o(m(idct_32x32_internal).pass2_end)] .pass2_loop: - mov [rsp+gprsize*3+16*35], r3 + mov [rsp+gprsize*3+16*35], r3d lea r3, [dstq+8] mov [rsp+gprsize*2+16*35], r3 mova m0, [coeffq+16*4 ] mova m1, [coeffq+16*12] @@ -5376,35 +5394,34 @@ .pass2_end1: lea tx2q, [o(m(idct_32x32_internal).pass2_end)] add coeffq, 16*32 mov dstq, [rsp+gprsize*2+16*35] - mov r3, [rsp+gprsize*3+16*35] - dec r3 + mov r3d, [rsp+gprsize*3+16*35] + dec r3d jg .pass2_loop ret cglobal inv_txfm_add_identity_identity_32x32, 4, 6, 8, 16*5, dst, stride, coeff, eob, tx2 %undef cmp - mov r4, 2 - mov r5, 4 + mov r4d, 2 cmp eobd, 136 - cmovge r4, r5 + mov r3d, 4 + cmovs r3d, r4d %if ARCH_X86_32 LEA r5, $$ %endif - lea r3, [dstq+8] - mov [rsp+gprsize*0+16*3], r3 - mov [rsp+gprsize*1+16*3], r4 - mov [rsp+gprsize*2+16*3], r4 + lea r4, [dstq+8] + mov [rsp+gprsize*0+16*3], r4 + mov [rsp+gprsize*1+16*3], r3d + mov [rsp+gprsize*2+16*3], r3d mov [rsp+gprsize*3+16*3], coeffq - mov r3, r4 .loop: LOAD_8ROWS coeffq, 64 mova [rsp+16*1], m6 lea tx2q, [o(m(idct_32x16_internal).end)] @@ -5420,24 +5437,24 @@ pxor m7, m7 REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 add coeffq, 16 - dec r3 + dec r3d jg .loop - mov r4, [rsp+gprsize*2+16*3] - dec r4 + mov r4d, [rsp+gprsize*2+16*3] + dec r4d jle .ret mov dstq, [rsp+gprsize*0+16*3] mov coeffq, [rsp+gprsize*3+16*3] mov [rsp+gprsize*2+16*3], r4 lea r3, [dstq+8] add coeffq, 64*8 mov [rsp+gprsize*0+16*3], r3 - mov r3, [rsp+gprsize*1+16*3] + mov r3d, [rsp+gprsize*1+16*3] mov [rsp+gprsize*3+16*3], coeffq jmp .loop .ret: RET @@ -5467,21 +5484,20 @@ cglobal idct_16x64_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp - mov r5, 4 - mov r4, 2 + mov r4d, 2 sub eobd, 151 - cmovge r4, r5 + mov [rsp+gprsize*1+16*67], eobd + mov r3d, 4 + cmovs r3d, r4d %if ARCH_X86_32 LEA r5, $$ %endif - mov [rsp+gprsize*1+16*67], eobd - mov r3, r4 mov [rsp+gprsize*2+16*67], coeffq .pass1_loop: LOAD_8ROWS coeffq+64*0, 64*2 call m(idct_8x8_internal).main @@ -5502,21 +5518,21 @@ .pass1_end1: SAVE_8ROWS coeffq+64*0, 64 add coeffq, 16 - dec r3 + dec r3d jg .pass1_loop mov coeffq, [rsp+gprsize*2+16*67] - mov r3, 2 + mov r3d, 2 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 lea r4, [o(m(idct_16x64_internal).end1)] .pass2_loop: - mov [rsp+gprsize*3+16*67], r3 + mov [rsp+gprsize*3+16*67], r3d mov eobd, [rsp+gprsize*1+16*67] mova m0, [coeffq+16*4 ] ;in1 mova m1, [coeffq+16*12] ;in3 mova m2, [coeffq+16*20] ;in5 @@ -5644,16 +5660,16 @@ .end2: add coeffq, 16*32 sub rsp, 16*32 mov dstq, [rsp+gprsize*2+16*67] - mov r3, [rsp+gprsize*3+16*67] + mov r3d, [rsp+gprsize*3+16*67] lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 lea r4, [o(m(idct_16x64_internal).end1)] - dec r3 + dec r3d jg .pass2_loop ret ALIGN function_align @@ -5671,15 +5687,13 @@ mova m1, [rsp+gprsize*2+16*37] ;in15 pmulhrsw m2, m1, [o(pw_3822x8)] ;t60,t61 pmulhrsw m1, [o(pw_m1474x8)] ;t34,t35 mova [rsp+gprsize*2+16*38], m1 ;t35 mova [rsp+gprsize*2+16*63], m2 ;t60 - pxor m6, m6 - psubw m3, m6, m1 - ITX_MULSUB_2W 3, 2, 0, 1, 7, 401, 4076 ;t34a, t61a - mova [rsp+gprsize*2+16*37], m3 ;t34a - mova [rsp+gprsize*2+16*64], m2 ;t61a + ITX_MULSUB_2W 2, 1, 0, 3, 7, m4076, 401 ;t34a, t61a + mova [rsp+gprsize*2+16*37], m2 ;t34a + mova [rsp+gprsize*2+16*64], m1 ;t61a mova m0, [rsp+gprsize*2+16*39] ;in9 pmulhrsw m3, m0, [o(pw_3996x8)] ;t58,t59 pmulhrsw m0, [o(pw_897x8)] ;t36,t37 mova [rsp+gprsize*2+16*39], m0 ;t36 @@ -5691,14 +5705,13 @@ mova m1, [rsp+gprsize*2+16*41] ;in7 pmulhrsw m2, m1, [o(pw_4036x8)] ;t56,t57 pmulhrsw m1, [o(pw_m700x8)] ;t38,t39 mova [rsp+gprsize*2+16*42], m1 ;t39 mova [rsp+gprsize*2+16*59], m2 ;t56 - psubw m3, m6, m1 - ITX_MULSUB_2W 3, 2, 0, 1, 7, 3166, 2598 ;t38a, t57a - mova [rsp+gprsize*2+16*41], m3 ;t38a - mova [rsp+gprsize*2+16*60], m2 ;t57a + ITX_MULSUB_2W 2, 1, 0, 3, 7, m2598, 3166 ;t38a, t57a + mova [rsp+gprsize*2+16*41], m2 ;t38a + mova [rsp+gprsize*2+16*60], m1 ;t57a mova m0, [rsp+gprsize*2+16*43] ;in5 pmulhrsw m3, m0, [o(pw_4065x8)] ;t54,t55 pmulhrsw m0, [o(pw_501x8)] ;t40,t41 mova [rsp+gprsize*2+16*43], m0 ;t40 @@ -5710,14 +5723,13 @@ mova m1, [rsp+gprsize*2+16*45] ;in11 pmulhrsw m2, m1, [o(pw_3948x8)] ;t52,t53 pmulhrsw m1, [o(pw_m1092x8)] ;t42,t43 mova [rsp+gprsize*2+16*46], m1 ;t43 mova [rsp+gprsize*2+16*55], m2 ;t52 - psubw m3, m6, m1 - ITX_MULSUB_2W 3, 2, 0, 1, 7, 1931, 3612 ;t42a, t53a - mova [rsp+gprsize*2+16*45], m3 ;t42a - mova [rsp+gprsize*2+16*56], m2 ;t53a + ITX_MULSUB_2W 2, 1, 0, 3, 7, m3612, 1931 ;t42a, t53a + mova [rsp+gprsize*2+16*45], m2 ;t42a + mova [rsp+gprsize*2+16*56], m1 ;t53a mova m0, [rsp+gprsize*2+16*47] ;in13 pmulhrsw m3, m0, [o(pw_3889x8)] ;t50,t51 pmulhrsw m0, [o(pw_1285x8)] ;t44,t45 mova m6, m0 @@ -5761,17 +5773,15 @@ pmulhrsw m1, [o(pw_m1474x8)] ;t35a psubsw m4, m1, m0 ;t34 paddsw m0, m1 ;t35 psubsw m5, m2, m3 ;t61 paddsw m3, m2 ;t60 - pxor m6, m6 - psubw m2, m6, m4 - ITX_MULSUB_2W 2, 5, 1, 4, 7, 401, 4076 ;t34a, t61a - mova [rsp+gprsize*2+16*37], m2 ;t34a + ITX_MULSUB_2W 5, 4, 1, 2, 7, m4076, 401 ;t34a, t61a + mova [rsp+gprsize*2+16*37], m5 ;t34a mova [rsp+gprsize*2+16*38], m0 ;t35 mova [rsp+gprsize*2+16*63], m3 ;t60 - mova [rsp+gprsize*2+16*64], m5 ;t61a + mova [rsp+gprsize*2+16*64], m4 ;t61a mova m0, [rsp+gprsize*2+16*39] ;in9 mova m1, [rsp+gprsize*2+16*61] ;in23 pmulhrsw m3, m0, [o(pw_3996x8)] ;t59a pmulhrsw m0, [o(pw_897x8)] ;t36a @@ -5795,16 +5805,15 @@ pmulhrsw m1, [o(pw_m700x8)] ;t39a psubsw m4, m1, m0 ;t38 paddsw m0, m1 ;t39 psubsw m5, m2, m3 ;t57 paddsw m3, m2 ;t56 - psubw m2, m6, m4 - ITX_MULSUB_2W 2, 5, 1, 4, 7, 3166, 2598 ;t38a, t57a - mova [rsp+gprsize*2+16*41], m2 ;t38a + ITX_MULSUB_2W 5, 4, 1, 2, 7, m2598, 3166 ;t38a, t57a + mova [rsp+gprsize*2+16*41], m5 ;t38a mova [rsp+gprsize*2+16*42], m0 ;t39 mova [rsp+gprsize*2+16*59], m3 ;t56 - mova [rsp+gprsize*2+16*60], m5 ;t57a + mova [rsp+gprsize*2+16*60], m4 ;t57a mova m0, [rsp+gprsize*2+16*43] ;in5 mova m1, [rsp+gprsize*2+16*57] ;in27 pmulhrsw m3, m0, [o(pw_4065x8)] ;t55a pmulhrsw m0, [o(pw_501x8)] ;t40a @@ -5828,16 +5837,15 @@ pmulhrsw m1, [o(pw_m1092x8)] ;t43a psubsw m4, m1, m0 ;t42 paddsw m0, m1 ;t43 psubsw m5, m2, m3 ;t53 paddsw m3, m2 ;t52 - psubw m2, m6, m4 - ITX_MULSUB_2W 2, 5, 1, 4, 7, 1931, 3612 ;t42a, t53a - mova [rsp+gprsize*2+16*45], m2 ;t42a + ITX_MULSUB_2W 5, 4, 1, 2, 7, m3612, 1931 ;t42a, t53a + mova [rsp+gprsize*2+16*45], m5 ;t42a mova [rsp+gprsize*2+16*46], m0 ;t43 mova [rsp+gprsize*2+16*55], m3 ;t52 - mova [rsp+gprsize*2+16*56], m5 ;t53a + mova [rsp+gprsize*2+16*56], m4 ;t53a mova m0, [rsp+gprsize*2+16*47] ;in13 mova m1, [rsp+gprsize*2+16*53] ;in19 pmulhrsw m3, m0, [o(pw_3889x8)] ;t51a pmulhrsw m0, [o(pw_1285x8)] ;t44a @@ -5864,39 +5872,33 @@ psubsw m4, m2, m3 ;t49 paddsw m3, m2 ;t48 ALIGN function_align .main2: - pxor m2, m2 - psubw m2, m5 - ITX_MULSUB_2W 2, 4, 1, 5, 7, 3920, 1189 ;t46a, t49a - + ITX_MULSUB_2W 4, 5, 1, 2, 7, m1189, 3920 ;t46a, t49a mova m1, [rsp+gprsize*2+16*54] ;t51 - psubsw m5, m0, m6 ;t44a + psubsw m2, m0, m6 ;t44a paddsw m0, m6 ;t47a psubsw m6, m3, m1 ;t51a paddsw m3, m1 ;t48a mova [rsp+gprsize*2+16*50], m0 ;t47a mova [rsp+gprsize*2+16*51], m3 ;t48a - pxor m1, m1 - psubw m3, m1, m5 - ITX_MULSUB_2W 3, 6, 0, 5, 7, 3406, 2276 ;t44, t51 - mova [rsp+gprsize*2+16*47], m3 ;t44 - mova [rsp+gprsize*2+16*54], m6 ;t51 + ITX_MULSUB_2W 6, 2, 0, 3, 7, m2276, 3406 ;t44, t51 + mova [rsp+gprsize*2+16*47], m6 ;t44 + mova [rsp+gprsize*2+16*54], m2 ;t51 mova m0, [rsp+gprsize*2+16*48] ;t45a mova m3, [rsp+gprsize*2+16*53] ;t50a - psubsw m5, m2, m0 ;t45 - paddsw m2, m0 ;t46 - psubsw m6, m4, m3 ;t50 - paddsw m4, m3 ;t49 - psubw m1, m5 - ITX_MULSUB_2W 1, 6, 0, 3, 7, 3406, 2276 ;t45a, t50a - mova [rsp+gprsize*2+16*48], m1 ;t45a - mova [rsp+gprsize*2+16*49], m2 ;t46 - mova [rsp+gprsize*2+16*52], m4 ;t49 - mova [rsp+gprsize*2+16*53], m6 ;t50a + psubsw m2, m4, m0 ;t45 + paddsw m4, m0 ;t46 + psubsw m6, m5, m3 ;t50 + paddsw m5, m3 ;t49 + ITX_MULSUB_2W 6, 2, 0, 3, 7, m2276, 3406 ;t45a, t50a + mova [rsp+gprsize*2+16*48], m6 ;t45a + mova [rsp+gprsize*2+16*49], m4 ;t46 + mova [rsp+gprsize*2+16*52], m5 ;t49 + mova [rsp+gprsize*2+16*53], m2 ;t50a mova m0, [rsp+gprsize*2+16*43] ;t40 mova m2, [rsp+gprsize*2+16*46] ;t43 mova m3, [rsp+gprsize*2+16*55] ;t52 mova m1, [rsp+gprsize*2+16*58] ;t55 @@ -5930,31 +5932,28 @@ mova m1, [rsp+gprsize*2+16*60] ;t57a psubsw m4, m0, m2 ;t37 paddsw m0, m2 ;t38 psubsw m5, m1, m3 ;t58 paddsw m1, m3 ;t57 - pxor m6, m6 - psubw m3, m6, m4 - ITX_MULSUB_2W 3, 5, 2, 4, 7, 799, 4017 ;t37a, t58a + ITX_MULSUB_2W 5, 4, 2, 3, 7, m4017, 799 ;t37a, t58a mova [rsp+gprsize*2+16*41], m0 ;t38 - mova [rsp+gprsize*2+16*40], m3 ;t37a - mova [rsp+gprsize*2+16*61], m5 ;t58a + mova [rsp+gprsize*2+16*40], m5 ;t37a + mova [rsp+gprsize*2+16*61], m4 ;t58a mova [rsp+gprsize*2+16*60], m1 ;t57 mova m0, [rsp+gprsize*2+16*42] ;t39 mova m2, [rsp+gprsize*2+16*39] ;t36 mova m3, [rsp+gprsize*2+16*62] ;t59 mova m1, [rsp+gprsize*2+16*59] ;t56 psubsw m4, m0, m2 ;t36a paddsw m0, m2 ;t39a psubsw m5, m1, m3 ;t59a paddsw m1, m3 ;t56a - psubw m3, m6, m4 - ITX_MULSUB_2W 3, 5, 2, 4, 7, 799, 4017 ;t36, t59 + ITX_MULSUB_2W 5, 4, 2, 3, 7, m4017, 799 ;t36, t59 mova [rsp+gprsize*2+16*42], m0 ;t39a - mova [rsp+gprsize*2+16*39], m3 ;t36 - mova [rsp+gprsize*2+16*62], m5 ;t59 + mova [rsp+gprsize*2+16*39], m5 ;t36 + mova [rsp+gprsize*2+16*62], m4 ;t59 mova [rsp+gprsize*2+16*59], m1 ;t56a mova m0, [rsp+gprsize*2+16*35] ;t32 mova m2, [rsp+gprsize*2+16*38] ;t35 mova m3, [rsp+gprsize*2+16*63] ;t60 @@ -6037,58 +6036,53 @@ mova m1, [rsp+gprsize*2+16*51] ;t48a psubsw m4, m0, m2 ;t40 paddsw m0, m2 ;t47 psubsw m5, m1, m3 ;t55 paddsw m1, m3 ;t48 - pxor m6, m6 - psubw m3, m6, m4 - ITX_MULSUB_2W 3, 5, 2, 4, 7, 1567, 3784 ;t40a, t55a + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t40a, t55a mova [rsp+gprsize*2+16*50], m0 ;t47 - mova [rsp+gprsize*2+16*43], m3 ;t40a - mova [rsp+gprsize*2+16*58], m5 ;t55a + mova [rsp+gprsize*2+16*43], m5 ;t40a + mova [rsp+gprsize*2+16*58], m4 ;t55a mova [rsp+gprsize*2+16*51], m1 ;t48 mova m0, [rsp+gprsize*2+16*49] ;t46 mova m2, [rsp+gprsize*2+16*44] ;t41 mova m3, [rsp+gprsize*2+16*57] ;t54 mova m1, [rsp+gprsize*2+16*52] ;t49 psubsw m4, m0, m2 ;t41a paddsw m0, m2 ;t46a psubsw m5, m1, m3 ;t54a paddsw m1, m3 ;t49a - psubw m3, m6, m4 - ITX_MULSUB_2W 3, 5, 2, 4, 7, 1567, 3784 ;t41, t54 + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t41, t54 mova [rsp+gprsize*2+16*49], m0 ;t46a - mova [rsp+gprsize*2+16*44], m3 ;t41 - mova [rsp+gprsize*2+16*57], m5 ;t54 + mova [rsp+gprsize*2+16*44], m5 ;t41 + mova [rsp+gprsize*2+16*57], m4 ;t54 mova [rsp+gprsize*2+16*52], m1 ;t49a mova m0, [rsp+gprsize*2+16*48] ;t45a mova m2, [rsp+gprsize*2+16*45] ;t42a mova m3, [rsp+gprsize*2+16*56] ;t53a mova m1, [rsp+gprsize*2+16*53] ;t50a psubsw m4, m0, m2 ;t42 paddsw m0, m2 ;t45 psubsw m5, m1, m3 ;t53 paddsw m1, m3 ;t50 - psubw m3, m6, m4 - ITX_MULSUB_2W 3, 5, 2, 4, 7, 1567, 3784 ;t42a, t53a + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t42a, t53a mova [rsp+gprsize*2+16*48], m0 ;t45 - mova [rsp+gprsize*2+16*45], m3 ;t42a - mova [rsp+gprsize*2+16*56], m5 ;t53a + mova [rsp+gprsize*2+16*45], m5 ;t42a + mova [rsp+gprsize*2+16*56], m4 ;t53a mova [rsp+gprsize*2+16*53], m1 ;t50 mova m0, [rsp+gprsize*2+16*47] ;t44 mova m2, [rsp+gprsize*2+16*46] ;t43 - mova m5, [rsp+gprsize*2+16*55] ;t52 + mova m3, [rsp+gprsize*2+16*55] ;t52 mova m1, [rsp+gprsize*2+16*54] ;t51 - psubsw m3, m0, m2 ;t43a + psubsw m4, m0, m2 ;t43a paddsw m0, m2 ;t44a - psubsw m4, m1, m5 ;t52a - paddsw m1, m5 ;t51a - psubw m5, m6, m3 - ITX_MULSUB_2W 5, 4, 2, 3, 7, 1567, 3784 ;t43, t52 + psubsw m5, m1, m3 ;t52a + paddsw m1, m3 ;t51a + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t43, t52 mova m2, [rsp+gprsize*2+16*38] ;t35a mova m3, [rsp+gprsize*2+16*31] ;tmp[28] psubsw m6, m2, m0 ;t44 paddsw m2, m0 ;t35 @@ -6641,21 +6635,20 @@ cglobal idct_32x64_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp - mov r5, 4 - mov r4, 2 + mov r4d, 2 sub eobd, 136 - cmovge r4, r5 + mov [rsp+gprsize*1+16*67], eobd + mov r3d, 4 + cmovs r3d, r4d %if ARCH_X86_32 LEA r5, $$ %endif - mov [rsp+gprsize*1+16*67], eobd - mov r3, r4 mov [rsp+gprsize*2+16*67], coeffq .pass1_loop: LOAD_8ROWS coeffq+64*1, 64*2, 1 mova [rsp+gprsize+16*19], m0 ;in1 @@ -6737,16 +6730,16 @@ .pass1_end4: SAVE_8ROWS coeffq+64*24, 64 add coeffq, 16 - dec r3 + dec r3d jg .pass1_loop .pass2: mov coeffq, [rsp+gprsize*2+16*67] - mov r3, 4 + mov r3d, 4 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 lea r4, [o(m(idct_16x64_internal).end1)] jmp m(idct_16x64_internal).pass2_loop @@ -6775,21 +6768,20 @@ RET cglobal idct_64x32_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp - mov r5, 4 - mov r4, 2 + mov r4d, 2 sub eobd, 136 - cmovge r4, r5 + mov [rsp+gprsize*1+16*67], eobd + mov r3d, 4 + cmovs r3d, r4d %if ARCH_X86_32 LEA r5, $$ %endif - mov [rsp+gprsize*1+16*67], eobd - mov r3, r4 mov [rsp+gprsize*2+16*67], coeffq mov [rsp+gprsize*3+16*67], dstq lea dstq, [rsp+gprsize+16*69] mov [rsp+gprsize*4+16*67], dstq @@ -6900,21 +6892,21 @@ .pass1_end7: SAVE_8ROWS dstq+64*24, 64 add coeffq, 16 add dstq, 16 - dec r3 + dec r3d jg .pass1_loop .pass2: mov coeffq, [rsp+gprsize*4+16*67] mov dstq, [rsp+gprsize*3+16*67] mov eobd, [rsp+gprsize*1+16*67] lea dstq, [dstq+32] mov [rsp+gprsize*1+16*35], eobd lea tx2q, [o(m(idct_64x32_internal).pass2_end)] - mov r3, 4 + mov r3d, 4 jmp m(idct_32x32_internal).pass2_loop .pass2_end: mova [rsp+gprsize+16*0], m7 lea r3, [o(m(idct_64x32_internal).pass2_end1)] @@ -6922,19 +6914,19 @@ .pass2_end1: lea tx2q, [o(m(idct_64x32_internal).pass2_end)] add coeffq, 16*32 mov dstq, [rsp+gprsize*2+16*35] - mov r3, [rsp+gprsize*3+16*35] - dec r3 + mov r3d, [rsp+gprsize*3+16*35] + dec r3d jg m(idct_32x32_internal).pass2_loop .pass2_end2: mov dstq, [rsp+gprsize*3+16*67] mov coeffq, [rsp+gprsize*2+16*67] lea tx2q, [o(m(idct_32x32_internal).pass2_end)] - mov r3, 4 + mov r3d, 4 jmp m(idct_32x32_internal).pass2_loop cglobal inv_txfm_add_dct_dct_64x64, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 @@ -6956,21 +6948,21 @@ jmp m(inv_txfm_add_dct_dct_64x16).body cglobal idct_64x64_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp - mov r5, 4 - mov r4, 2 + mov r5d, 4 + mov r4d, 2 sub eobd, 136 - cmovge r4, r5 + cmovns r4d, r5d %if ARCH_X86_32 LEA r5, $$ %endif mov [rsp+gprsize*1+16*67], eobd - mov r3, r4 + mov r3d, r4d mov [rsp+gprsize*4+16*67], coeffq mov [rsp+gprsize*3+16*67], dstq lea dstq, [rsp+gprsize+16*69] mov [rsp+gprsize*2+16*67], dstq @@ -7089,18 +7081,18 @@ .pass1_end7: SAVE_8ROWS dstq+64*24, 64 add coeffq, 16 add dstq, 16 - dec r3 + dec r3d jg .pass1_loop .pass2: mov dstq, [rsp+gprsize*3+16*67] mov coeffq, [rsp+gprsize*2+16*67] lea dstq, [dstq+32] - mov r3, 4 + mov r3d, 4 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 lea r4, [o(m(idct_64x64_internal).pass2_end)] jmp m(idct_16x64_internal).pass2_loop @@ -7115,22 +7107,22 @@ .pass2_end1: add coeffq, 16*32 sub rsp, 16*32 mov dstq, [rsp+gprsize*2+16*67] - mov r3, [rsp+gprsize*3+16*67] + mov r3d, [rsp+gprsize*3+16*67] lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 lea r4, [o(m(idct_64x64_internal).pass2_end)] - dec r3 + dec r3d jg m(idct_16x64_internal).pass2_loop .pass2_end2: mov coeffq, [rsp+gprsize*4+16*67] mov dstq, [rsp+gprsize*2+16*67] - mov r3, 4 + mov r3d, 4 sub dstq, 72 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 lea r4, [o(m(idct_16x64_internal).end1)] jmp m(idct_16x64_internal).pass2_loop diff --git a/third_party/dav1d/src/x86/looprestoration.asm b/third_party/dav1d/src/x86/looprestoration.asm --- a/third_party/dav1d/src/x86/looprestoration.asm +++ b/third_party/dav1d/src/x86/looprestoration.asm @@ -167,18 +167,25 @@ pmullw m5, m12 paddw m0, m7 paddw m2, m4 paddw m0, m6 paddw m2, m5 - paddsw m0, m8 + ; for a signed overflow to happen we need filter and pixels as follow: + ; filter => -5,-23,-17,90,-17,-23,-5 + ; pixels => 255,255,255,0,255,255,255 or 0,0,0,255,0,0,0 + ; m0 would fall in the range [-59A6;+59A6] = [A65A;59A6] + ; m8 would fall in the range [-3FFC;+3F84] = [C004;3F84] + ; 32-bit arithmetic m0+m8 = [-99A2;+992A] = [FFFF665E;992A] + ; => signed 16-bit overflow occurs + paddsw m0, m8 ; paddsw clips this range to [-8000;+7FFF] paddsw m2, m3 - psraw m0, 3 + psraw m0, 3 ; shift changes the range to [-1000;+FFF] psraw m2, 3 - paddw m0, m11 - paddw m2, m11 - mova [dstptrq], xm0 - mova [dstptrq+16], xm2 + paddw m0, m11 ; adding back 800 (removed in m8) changes the + paddw m2, m11 ; range to [-800;+17FF] as defined in the spec + mova [dstptrq], xm0 ; (note that adding another 800 would give us + mova [dstptrq+16], xm2; the same range as in the C code => [0;1FFF]) vextracti128 [dstptrq+32], m0, 1 vextracti128 [dstptrq+48], m2, 1 vextracti128 xm0, m1, 1 add srcptrq, 32 add dstptrq, 64 @@ -345,11 +352,11 @@ pinsrw xm0, [srcq+xq-2], 7 .expand_x: punpckhbw xm0, xm1 ; when we reach this, xm0 contains left two px in highest words - cmp xq, -16 + cmp xd, -16 jle .loop_x .partial_load_and_extend: vpbroadcastb m3, [srcq-1] pmovzxbw m2, [srcq+xq] punpcklbw m3, m1 @@ -394,21 +401,21 @@ ; if x <= -16 we can reload more pixels ; else if x < 0 we reload and extend (this implies have_right=0) ; else if x < xlimd we extend from previous load (this implies have_right=0) ; else we are done - cmp xq, -16 + cmp xd, -16 jle .loop_x - test xq, xq + test xd, xd jl .partial_load_and_extend - cmp xq, xlimq + cmp xd, xlimd jl .right_extend add sumsqq, (384+16)*4 add sumq, (384+16)*2 add srcq, strideq - dec hd + dec hd jg .loop_y RET INIT_YMM avx2 cglobal sgr_box3_v, 5, 10, 9, sumsq, sum, w, h, edge, x, y, sumsq_ptr, sum_ptr, ylim @@ -416,11 +423,11 @@ mov ylimd, edged and ylimd, 8 ; have_bottom shr ylimd, 2 sub ylimd, 2 ; -2 if have_bottom=0, else 0 .loop_x: - lea yd, [hd+ylimd+2] + lea yd, [hq+ylimq+2] lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] lea sum_ptrq, [sumq+xq*2+2-(384+16)*2] test edged, 4 ; have_top jnz .load_top movu m0, [sumsq_ptrq+(384+16)*4*1] @@ -718,13 +725,13 @@ pinsrd xm0, [srcq+xq-4], 3 .expand_x: punpckhbw xm0, xm1 ; when we reach this, xm0 contains left two px in highest words - cmp xq, -16 + cmp xd, -16 jle .loop_x - test xq, xq + test xd, xd jge .right_extend .partial_load_and_extend: vpbroadcastb m3, [srcq-1] pmovzxbw m2, [srcq+xq] punpcklbw m3, m1 @@ -779,15 +786,15 @@ ; if x <= -16 we can reload more pixels ; else if x < 0 we reload and extend (this implies have_right=0) ; else if x < xlimd we extend from previous load (this implies have_right=0) ; else we are done - cmp xq, -16 + cmp xd, -16 jle .loop_x - test xq, xq + test xd, xd jl .partial_load_and_extend - cmp xq, xlimq + cmp xd, xlimd jl .right_extend add sumsqq, (384+16)*4 add sumq, (384+16)*2 add srcq, strideq @@ -801,11 +808,11 @@ mov ylimd, edged and ylimd, 8 ; have_bottom shr ylimd, 2 sub ylimd, 3 ; -3 if have_bottom=0, else -1 .loop_x: - lea yd, [hd+ylimd+2] + lea yd, [hq+ylimq+2] lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] lea sum_ptrq, [sumq+xq*2+2-(384+16)*2] test edged, 4 ; have_top jnz .load_top movu m0, [sumsq_ptrq+(384+16)*4*1] diff --git a/third_party/dav1d/src/x86/looprestoration_ssse3.asm b/third_party/dav1d/src/x86/looprestoration_ssse3.asm --- a/third_party/dav1d/src/x86/looprestoration_ssse3.asm +++ b/third_party/dav1d/src/x86/looprestoration_ssse3.asm @@ -357,12 +357,12 @@ pmullw m5, m12 paddw m0, m7 paddw m2, m4 paddw m0, m3 paddw m2, m5 - paddsw m0, m8 - paddsw m2, m6 + paddsw m0, m8 ; see the avx2 for an explanation + paddsw m2, m6 ; of how the clipping works here psraw m0, 3 psraw m2, 3 paddw m0, m11 paddw m2, m11 mova [dstptrq+ 0], m0 @@ -723,11 +723,11 @@ pslldq m0, 14 .expand_x: punpckhbw xm0, xm1 ; when we reach this, m0 contains left two px in highest words - cmp xq, -8 + cmp xd, -8 jle .loop_x .partial_load_and_extend: movd m3, [srcq-4] pshufb m3, [PIC_sym(pb_3)] movq m2, [srcq+xq] @@ -1297,13 +1297,13 @@ pslldq m0, 12 .expand_x: punpckhbw m0, m1 ; when we reach this, m0 contains left two px in highest words - cmp xq, -8 + cmp xd, -8 jle .loop_x - test xq, xq + test xd, xd jge .right_extend .partial_load_and_extend: XCHG_PIC_REG movd m3, [srcq-1] movq m2, [srcq+xq] @@ -1392,15 +1392,15 @@ ; if x <= -8 we can reload more pixels ; else if x < 0 we reload and extend (this implies have_right=0) ; else if x < xlimd we extend from previous load (this implies have_right=0) ; else we are done - cmp xq, -8 + cmp xd, -8 jle .loop_x - test xq, xq + test xd, xd jl .partial_load_and_extend - cmp xq, xlimq + cmp xd, xlimd jl .right_extend add sumsqq, (384+16)*4 add sumq, (384+16)*2 add srcq, strideq diff --git a/third_party/dav1d/src/x86/mc.asm b/third_party/dav1d/src/x86/mc.asm --- a/third_party/dav1d/src/x86/mc.asm +++ b/third_party/dav1d/src/x86/mc.asm @@ -25,11 +25,11 @@ %include "ext/x86/x86inc.asm" %if ARCH_X86_64 -SECTION_RODATA 32 +SECTION_RODATA 64 ; dav1d_obmc_masks[] with 64-x interleaved obmc_masks: db 0, 0, 0, 0 ; 2 db 45, 19, 64, 0 @@ -44,59 +44,140 @@ db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0 -warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 - db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 -warp_8x8_shufB: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 - db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 -subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 - db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 -subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 -subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 -subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 -bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 -deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 -blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +bidir_sctr_w4: dd 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +wm_420_perm4: db 1, 3, 9, 11, 5, 7, 13, 15, 17, 19, 25, 27, 21, 23, 29, 31 + db 33, 35, 41, 43, 37, 39, 45, 47, 49, 51, 57, 59, 53, 55, 61, 63 + db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30 + db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62 +wm_420_perm8: db 1, 3, 17, 19, 5, 7, 21, 23, 9, 11, 25, 27, 13, 15, 29, 31 + db 33, 35, 49, 51, 37, 39, 53, 55, 41, 43, 57, 59, 45, 47, 61, 63 + db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30 + db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62 +wm_420_perm16: db 1, 3, 33, 35, 5, 7, 37, 39, 9, 11, 41, 43, 13, 15, 45, 47 + db 17, 19, 49, 51, 21, 23, 53, 55, 25, 27, 57, 59, 29, 31, 61, 63 + db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46 + db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62 +wm_420_mask: db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 + db 67, 71, 75, 79, 83, 87, 91, 95, 99,103,107,111,115,119,123,127 + db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 +wm_422_mask: db 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62 + db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 66, 70, 74, 78, 82, 86, 90, 94, 98,102,106,110,114,118,122,126 + db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 +wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 + db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 +bilin_h_perm16: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 + db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 + db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39 + db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47 +bilin_h_perm32: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 + db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 + db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23 + db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31 +bilin_v_perm8: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 + db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 + db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87 + db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39 +bilin_v_perm16: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 + db 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 + db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23 + db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31 +bilin_v_perm32: db 64, 0, 65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7 + db 72, 8, 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15 + db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 + db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31 +bilin_v_perm64: dq 0, 4, 1, 5, 2, 6, 3, 7 +spel_h_perm16a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 + db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 + db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 +spel_h_perm16b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 + db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 + db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42 + db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50 +spel_h_perm16c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 + db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54 +spel_h_perm32a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 + db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 +spel_h_perm32b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 + db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 + db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26 + db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34 +spel_h_perm32c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 + db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 +spel_hv_perm4a: db 8, 9, 16, 17, 10, 11, 18, 19, 12, 13, 20, 21, 14, 15, 22, 23 + db 16, 17, 24, 25, 18, 19, 26, 27, 20, 21, 28, 29, 22, 23, 30, 31 +spel_hv_perm4b: db 24, 25, 32, 33, 26, 27, 34, 35, 28, 29, 36, 37, 30, 31, 38, 39 + db 32, 33, 40, 41, 34, 35, 42, 43, 36, 37, 44, 45, 38, 39, 46, 47 + db 40, 41, 48, 49, 42, 43, 50, 51, 44, 45, 52, 53, 46, 47, 54, 55 + db 48, 49, 56, 57, 50, 51, 58, 59, 52, 53, 60, 61, 54, 55, 62, 63 -pb_64: times 4 db 64 +warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 + db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 +warp_8x8_shufB: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 + db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 +subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 + db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 +subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_v_shuf4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 +bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 +bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 +deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +pb_8x0_8x8: times 8 db 0 + times 8 db 8 + +ALIGN 32 +resize_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 +resize_shuf: times 5 db 0 + db 1, 2, 3, 4, 5, 6 + times 5+8 db 7 + +ALIGN 8 +wm_420_perm64: dq 0xfedcba9876543210 +wm_420_sign: dd 0x01020102, 0x01010101 +wm_422_sign: dd 0x80808080, 0x7f7f7f7f +wm_sign_avx512: dd 0x40804080, 0xc0c0c0c0, 0x40404040 + +ALIGN 4 +pw_m128 times 2 dw -128 +pw_m256: times 2 dw -256 pw_34: times 2 dw 34 pw_258: times 2 dw 258 pw_512: times 2 dw 512 pw_1024: times 2 dw 1024 pw_2048: times 2 dw 2048 pw_6903: times 2 dw 6903 pw_8192: times 2 dw 8192 +pd_2: dd 2 pd_32: dd 32 +pd_63: dd 63 pd_512: dd 512 pd_32768: dd 32768 -cextern mc_subpel_filters -%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) +%define pb_m64 (wm_sign_avx512+4) +%define pb_64 (wm_sign_avx512+8) +%define pb_127 (wm_422_sign +4) -%macro BIDIR_JMP_TABLE 1-* - %xdefine %1_table (%%table - 2*%2) - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) - %%table: - %rep %0 - 1 - dd %%prefix %+ .w%2 - %%base - %rotate 1 - %endrep -%endmacro - -BIDIR_JMP_TABLE avg_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_avg_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE mask_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_420_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_422_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_444_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE blend_avx2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_v_avx2, 2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_h_avx2, 2, 4, 8, 16, 32, 32, 32 +cextern mc_subpel_filters +cextern mc_warp_filter +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) %macro BASE_JMP_TABLE 3-* %xdefine %1_%2_table (%%table - %3) %xdefine %%base %1_%2 %%table: @@ -104,16 +185,10 @@ dw %%base %+ _w%3 - %%base %rotate 1 %endrep %endmacro -%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_avx2.put) -%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_avx2.prep) - -BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 -BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 - %macro HV_JMP_TABLE 5-* %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) %xdefine %%base %1_%3 %assign %%types %4 %if %%types & 1 @@ -142,18 +217,54 @@ %rotate 1 %endrep %endif %endmacro -HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 +%macro BIDIR_JMP_TABLE 1-* + %xdefine %1_table (%%table - 2*%2) + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + %%table: + %rep %0 - 1 + dd %%prefix %+ .w%2 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_avx2.put) +%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_avx2.prep) +%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_avx512icl.prep) %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX -cextern mc_warp_filter +BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend_avx2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v_avx2, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h_avx2, 2, 4, 8, 16, 32, 32, 32 + +%if HAVE_AVX512ICL +BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444_avx512icl, 4, 8, 16, 32, 64, 128 +%endif ; HAVE_AVX512ICL SECTION .text INIT_XMM avx2 DECLARE_REG_TMP 4, 6, 7 @@ -539,11 +650,11 @@ mov t0, dstq mov t1, srcq lea t2d, [hq+(3<<8)] .v_w128_loop: PUT_BILIN_V_W32 - mov hb, t2b + movzx hd, t2b add t0, 32 add t1, 32 mov dstq, t0 mov srcq, t1 sub t2d, 1<<8 @@ -727,14 +838,15 @@ jmp .hv_w32gt .hv_w128: lea t2d, [hq+(3<<8)] jmp .hv_w32gt +%macro PREP_BILIN 0 DECLARE_REG_TMP 3, 5, 6 cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 movifnidn mxyd, r5m ; mx - lea t2, [prep_avx2] + lea t2, [prep%+SUFFIX] tzcnt wd, wm movifnidn hd, hm test mxyd, mxyd jnz .h mov mxyd, r6m ; my @@ -749,174 +861,256 @@ movd xm0, [srcq+strideq*0] pinsrd xm0, [srcq+strideq*1], 1 pinsrd xm0, [srcq+strideq*2], 2 pinsrd xm0, [srcq+stride3q ], 3 lea srcq, [srcq+strideq*4] - pmovzxbw m0, xm0 - psllw m0, 4 - mova [tmpq], m0 + pmovzxbw ym0, xm0 + psllw ym0, 4 + mova [tmpq], ym0 add tmpq, 32 sub hd, 4 jg .prep_w4 RET .prep_w8: movq xm0, [srcq+strideq*0] +%if cpuflag(avx512) + movq xm1, [srcq+strideq*1] + vinserti128 ym0, [srcq+strideq*2], 1 + vinserti128 ym1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + punpcklqdq ym0, ym1 + pmovzxbw m0, ym0 + psllw m0, 4 + mova [tmpq], m0 +%else movhps xm0, [srcq+strideq*1] movq xm1, [srcq+strideq*2] movhps xm1, [srcq+stride3q ] lea srcq, [srcq+strideq*4] pmovzxbw m0, xm0 pmovzxbw m1, xm1 psllw m0, 4 psllw m1, 4 mova [tmpq+32*0], m0 mova [tmpq+32*1], m1 +%endif add tmpq, 32*2 sub hd, 4 jg .prep_w8 RET .prep_w16: +%if cpuflag(avx512) + movu xm0, [srcq+strideq*0] + vinserti128 ym0, [srcq+strideq*1], 1 + movu xm1, [srcq+strideq*2] + vinserti128 ym1, [srcq+stride3q ], 1 + pmovzxbw m0, ym0 + pmovzxbw m1, ym1 +%else + pmovzxbw m0, [srcq+strideq*0] + pmovzxbw m1, [srcq+strideq*1] + pmovzxbw m2, [srcq+strideq*2] + pmovzxbw m3, [srcq+stride3q ] +%endif + lea srcq, [srcq+strideq*4] + psllw m0, 4 + psllw m1, 4 +%if notcpuflag(avx512) + psllw m2, 4 + psllw m3, 4 +%endif + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m1 +%if notcpuflag(avx512) + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 +%endif + add tmpq, 32*4 + sub hd, 4 + jg .prep_w16 + RET +.prep_w32: +%if cpuflag(avx512) pmovzxbw m0, [srcq+strideq*0] pmovzxbw m1, [srcq+strideq*1] pmovzxbw m2, [srcq+strideq*2] pmovzxbw m3, [srcq+stride3q ] lea srcq, [srcq+strideq*4] - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 - add tmpq, 32*4 - sub hd, 4 - jg .prep_w16 - RET -.prep_w32: +%else pmovzxbw m0, [srcq+strideq*0+16*0] pmovzxbw m1, [srcq+strideq*0+16*1] pmovzxbw m2, [srcq+strideq*1+16*0] pmovzxbw m3, [srcq+strideq*1+16*1] lea srcq, [srcq+strideq*2] +%endif psllw m0, 4 psllw m1, 4 psllw m2, 4 psllw m3, 4 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 - add tmpq, 32*4 - sub hd, 2 + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m1 + mova [tmpq+mmsize*2], m2 + mova [tmpq+mmsize*3], m3 + add tmpq, mmsize*4 + sub hd, mmsize*4/(32*2) jg .prep_w32 RET .prep_w64: +%if cpuflag(avx512) + pmovzxbw m0, [srcq+strideq*0+32*0] + pmovzxbw m1, [srcq+strideq*0+32*1] + pmovzxbw m2, [srcq+strideq*1+32*0] + pmovzxbw m3, [srcq+strideq*1+32*1] + lea srcq, [srcq+strideq*2] +%else pmovzxbw m0, [srcq+16*0] pmovzxbw m1, [srcq+16*1] pmovzxbw m2, [srcq+16*2] pmovzxbw m3, [srcq+16*3] add srcq, strideq +%endif psllw m0, 4 psllw m1, 4 psllw m2, 4 psllw m3, 4 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 - add tmpq, 32*4 + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m1 + mova [tmpq+mmsize*2], m2 + mova [tmpq+mmsize*3], m3 + add tmpq, mmsize*4 +%if cpuflag(avx512) + sub hd, 2 +%else dec hd +%endif jg .prep_w64 RET .prep_w128: - pmovzxbw m0, [srcq+16*0] - pmovzxbw m1, [srcq+16*1] - pmovzxbw m2, [srcq+16*2] - pmovzxbw m3, [srcq+16*3] + pmovzxbw m0, [srcq+(mmsize/2)*0] + pmovzxbw m1, [srcq+(mmsize/2)*1] + pmovzxbw m2, [srcq+(mmsize/2)*2] + pmovzxbw m3, [srcq+(mmsize/2)*3] psllw m0, 4 psllw m1, 4 psllw m2, 4 psllw m3, 4 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m1 + mova [tmpq+mmsize*2], m2 + mova [tmpq+mmsize*3], m3 +%if notcpuflag(avx512) pmovzxbw m0, [srcq+16*4] pmovzxbw m1, [srcq+16*5] pmovzxbw m2, [srcq+16*6] pmovzxbw m3, [srcq+16*7] +%endif add tmpq, 32*8 add srcq, strideq +%if notcpuflag(avx512) psllw m0, 4 psllw m1, 4 psllw m2, 4 psllw m3, 4 mova [tmpq-32*4], m0 mova [tmpq-32*3], m1 mova [tmpq-32*2], m2 mova [tmpq-32*1], m3 +%endif dec hd jg .prep_w128 RET .h: ; 16 * src[x] + (mx * (src[x + 1] - src[x])) ; = (16 - mx) * src[x] + mx * src[x + 1] imul mxyd, 0xff01 - vbroadcasti128 m4, [bilin_h_shuf8] add mxyd, 16 << 8 +%if cpuflag(avx512) + vpbroadcastw m5, mxyd +%else movd xm5, mxyd + vbroadcasti128 m4, [bilin_h_shuf8] + vpbroadcastw m5, xm5 +%endif mov mxyd, r6m ; my - vpbroadcastw m5, xm5 test mxyd, mxyd jnz .hv movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)] add wq, t2 lea stride3q, [strideq*3] jmp wq .h_w4: - vbroadcasti128 m4, [bilin_h_shuf4] + vbroadcasti128 ym4, [bilin_h_shuf4] .h_w4_loop: movq xm0, [srcq+strideq*0] movhps xm0, [srcq+strideq*1] movq xm1, [srcq+strideq*2] movhps xm1, [srcq+stride3q ] lea srcq, [srcq+strideq*4] - vinserti128 m0, m0, xm1, 1 - pshufb m0, m4 - pmaddubsw m0, m5 - mova [tmpq], m0 + vinserti128 ym0, xm1, 1 + pshufb ym0, ym4 + pmaddubsw ym0, ym5 + mova [tmpq], ym0 add tmpq, 32 sub hd, 4 jg .h_w4_loop RET .h_w8: - movu xm0, [srcq+strideq*0] - vinserti128 m0, m0, [srcq+strideq*1], 1 - movu xm1, [srcq+strideq*2] - vinserti128 m1, m1, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] +%if cpuflag(avx512) + vbroadcasti128 m4, [bilin_h_shuf8] +.h_w8_loop: + movu xm0, [srcq+strideq*0] + vinserti128 ym0, [srcq+strideq*1], 1 + vinserti128 m0, [srcq+strideq*2], 2 + vinserti128 m0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pshufb m0, m4 + pmaddubsw m0, m5 + mova [tmpq+64*0], m0 +%else +.h_w8_loop: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + movu xm1, [srcq+strideq*2] + vinserti128 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] pshufb m0, m4 pshufb m1, m4 pmaddubsw m0, m5 pmaddubsw m1, m5 mova [tmpq+32*0], m0 mova [tmpq+32*1], m1 +%endif add tmpq, 32*2 sub hd, 4 - jg .h_w8 + jg .h_w8_loop RET .h_w16: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 - movu xm1, [srcq+strideq*1+8*0] - vinserti128 m1, m1, [srcq+strideq*1+8*1], 1 - movu xm2, [srcq+strideq*2+8*0] - vinserti128 m2, m2, [srcq+strideq*2+8*1], 1 - movu xm3, [srcq+stride3q +8*0] - vinserti128 m3, m3, [srcq+stride3q +8*1], 1 - lea srcq, [srcq+strideq*4] +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm16] +.h_w16_loop: + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 + movu ym1, [srcq+strideq*2] + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vpermb m0, m4, m0 + vpermb m1, m4, m1 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 +%else +.h_w16_loop: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + movu xm2, [srcq+strideq*2+8*0] + vinserti128 m2, [srcq+strideq*2+8*1], 1 + movu xm3, [srcq+stride3q +8*0] + vinserti128 m3, [srcq+stride3q +8*1], 1 + lea srcq, [srcq+strideq*4] pshufb m0, m4 pshufb m1, m4 pshufb m2, m4 pshufb m3, m4 pmaddubsw m0, m5 @@ -925,97 +1119,137 @@ pmaddubsw m3, m5 mova [tmpq+32*0], m0 mova [tmpq+32*1], m1 mova [tmpq+32*2], m2 mova [tmpq+32*3], m3 +%endif add tmpq, 32*4 sub hd, 4 - jg .h_w16 + jg .h_w16_loop RET .h_w32: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 - movu xm1, [srcq+strideq*0+8*2] - vinserti128 m1, m1, [srcq+strideq*0+8*3], 1 - movu xm2, [srcq+strideq*1+8*0] - vinserti128 m2, m2, [srcq+strideq*1+8*1], 1 - movu xm3, [srcq+strideq*1+8*2] - vinserti128 m3, m3, [srcq+strideq*1+8*3], 1 - lea srcq, [srcq+strideq*2] +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm32] +.h_w32_loop: + vpermb m0, m4, [srcq+strideq*0] + vpermb m1, m4, [srcq+strideq*1] + vpermb m2, m4, [srcq+strideq*2] + vpermb m3, m4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] +%else +.h_w32_loop: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + movu xm1, [srcq+strideq*0+8*2] + vinserti128 m1, [srcq+strideq*0+8*3], 1 + movu xm2, [srcq+strideq*1+8*0] + vinserti128 m2, [srcq+strideq*1+8*1], 1 + movu xm3, [srcq+strideq*1+8*2] + vinserti128 m3, [srcq+strideq*1+8*3], 1 + lea srcq, [srcq+strideq*2] pshufb m0, m4 pshufb m1, m4 pshufb m2, m4 pshufb m3, m4 +%endif pmaddubsw m0, m5 pmaddubsw m1, m5 pmaddubsw m2, m5 pmaddubsw m3, m5 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 - add tmpq, 32*4 - sub hd, 2 - jg .h_w32 + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m1 + mova [tmpq+mmsize*2], m2 + mova [tmpq+mmsize*3], m3 + add tmpq, mmsize*4 + sub hd, mmsize*4/(32*2) + jg .h_w32_loop RET .h_w64: - movu xm0, [srcq+8*0] - vinserti128 m0, m0, [srcq+8*1], 1 - movu xm1, [srcq+8*2] - vinserti128 m1, m1, [srcq+8*3], 1 - movu xm2, [srcq+8*4] - vinserti128 m2, m2, [srcq+8*5], 1 - movu xm3, [srcq+8*6] - vinserti128 m3, m3, [srcq+8*7], 1 +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm32] +.h_w64_loop: + vpermb m0, m4, [srcq+strideq*0+32*0] + vpermb m1, m4, [srcq+strideq*0+32*1] + vpermb m2, m4, [srcq+strideq*1+32*0] + vpermb m3, m4, [srcq+strideq*1+32*1] + lea srcq, [srcq+strideq*2] +%else +.h_w64_loop: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + movu xm2, [srcq+8*4] + vinserti128 m2, [srcq+8*5], 1 + movu xm3, [srcq+8*6] + vinserti128 m3, [srcq+8*7], 1 add srcq, strideq pshufb m0, m4 pshufb m1, m4 pshufb m2, m4 pshufb m3, m4 +%endif pmaddubsw m0, m5 pmaddubsw m1, m5 pmaddubsw m2, m5 pmaddubsw m3, m5 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 - add tmpq, 32*4 + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m1 + mova [tmpq+mmsize*2], m2 + mova [tmpq+mmsize*3], m3 + add tmpq, mmsize*4 +%if cpuflag(avx512) + sub hd, 2 +%else dec hd - jg .h_w64 +%endif + jg .h_w64_loop RET .h_w128: - movu xm0, [srcq+8*0] - vinserti128 m0, m0, [srcq+8*1], 1 - movu xm1, [srcq+8*2] - vinserti128 m1, m1, [srcq+8*3], 1 - movu xm2, [srcq+8*4] - vinserti128 m2, m2, [srcq+8*5], 1 - movu xm3, [srcq+8*6] - vinserti128 m3, m3, [srcq+8*7], 1 +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm32] +.h_w128_loop: + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + vpermb m2, m4, [srcq+32*2] + vpermb m3, m4, [srcq+32*3] +%else +.h_w128_loop: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + movu xm2, [srcq+8*4] + vinserti128 m2, [srcq+8*5], 1 + movu xm3, [srcq+8*6] + vinserti128 m3, [srcq+8*7], 1 pshufb m0, m4 pshufb m1, m4 pshufb m2, m4 pshufb m3, m4 +%endif pmaddubsw m0, m5 pmaddubsw m1, m5 pmaddubsw m2, m5 pmaddubsw m3, m5 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 - movu xm0, [srcq+8* 8] - vinserti128 m0, m0, [srcq+8* 9], 1 - movu xm1, [srcq+8*10] - vinserti128 m1, m1, [srcq+8*11], 1 - movu xm2, [srcq+8*12] - vinserti128 m2, m2, [srcq+8*13], 1 - movu xm3, [srcq+8*14] - vinserti128 m3, m3, [srcq+8*15], 1 + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m1 + mova [tmpq+mmsize*2], m2 + mova [tmpq+mmsize*3], m3 +%if notcpuflag(avx512) + movu xm0, [srcq+8* 8] + vinserti128 m0, [srcq+8* 9], 1 + movu xm1, [srcq+8*10] + vinserti128 m1, [srcq+8*11], 1 + movu xm2, [srcq+8*12] + vinserti128 m2, [srcq+8*13], 1 + movu xm3, [srcq+8*14] + vinserti128 m3, [srcq+8*15], 1 +%endif add tmpq, 32*8 add srcq, strideq +%if notcpuflag(avx512) pshufb m0, m4 pshufb m1, m4 pshufb m2, m4 pshufb m3, m4 pmaddubsw m0, m5 @@ -1024,24 +1258,43 @@ pmaddubsw m3, m5 mova [tmpq-32*4], m0 mova [tmpq-32*3], m1 mova [tmpq-32*2], m2 mova [tmpq-32*1], m3 +%endif dec hd - jg .h_w128 + jg .h_w128_loop RET .v: WIN64_SPILL_XMM 7 movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] imul mxyd, 0xff01 add mxyd, 16 << 8 add wq, t2 lea stride3q, [strideq*3] +%if cpuflag(avx512) + vpbroadcastw m6, mxyd +%else movd xm6, mxyd vpbroadcastw m6, xm6 +%endif jmp wq .v_w4: +%if cpuflag(avx512) + vpbroadcastd xm0, [srcq+strideq*0] + mov r3d, 0x29 + vbroadcasti128 ym3, [bilin_v_shuf4] + kmovb k1, r3d +.v_w4_loop: + vpblendmd xm1{k1}, xm0, [srcq+strideq*1] {1to4} ; __01 ____ + vpbroadcastd ym2, [srcq+strideq*2] + vpbroadcastd ym2{k1}, [srcq+stride3q ] ; __2_ 23__ + lea srcq, [srcq+strideq*4] + vpbroadcastd ym0, [srcq+strideq*0] + punpckhqdq ym2{k1}, ym1, ym0 ; 012_ 234_ + pshufb ym2, ym3 +%else movd xm0, [srcq+strideq*0] .v_w4_loop: vpbroadcastd m1, [srcq+strideq*2] vpbroadcastd xm2, [srcq+strideq*1] vpbroadcastd m3, [srcq+stride3q ] @@ -1051,19 +1304,35 @@ vpblendd m3, m3, m2, 0x0f ; 1 1 3 3 vpblendd m2, m1, m0, 0xa0 ; 0 2 2 4 vpblendd m1, m1, m3, 0xaa ; 0 1 2 3 vpblendd m2, m2, m3, 0x55 ; 1 2 3 4 punpcklbw m2, m1 - pmaddubsw m2, m6 - mova [tmpq], m2 +%endif + pmaddubsw ym2, ym6 + mova [tmpq], ym2 add tmpq, 32 sub hd, 4 jg .v_w4_loop RET .v_w8: +%if cpuflag(avx512icl) + mova m5, [bilin_v_perm8] + vbroadcasti128 ym0, [srcq+strideq*0] +%else movq xm0, [srcq+strideq*0] +%endif .v_w8_loop: +%if cpuflag(avx512icl) + vinserti128 ym1, ym0, [srcq+strideq*1], 1 + vpbroadcastq ym0, [srcq+strideq*2] + vinserti128 m1, [srcq+stride3q ], 2 + lea srcq, [srcq+strideq*4] + vinserti128 ym0, [srcq+strideq*0], 0 + vpermt2b m1, m5, m0 + pmaddubsw m1, m6 + mova [tmpq], m1 +%else vpbroadcastq m1, [srcq+strideq*2] vpbroadcastq m2, [srcq+strideq*1] vpbroadcastq m3, [srcq+stride3q ] lea srcq, [srcq+strideq*4] vpblendd m1, m1, m0, 0x03 ; 0 2 2 2 @@ -1076,25 +1345,42 @@ punpckhbw m2, m1 pmaddubsw m3, m6 pmaddubsw m2, m6 mova [tmpq+32*0], m3 mova [tmpq+32*1], m2 +%endif add tmpq, 32*2 sub hd, 4 jg .v_w8_loop RET .v_w16: +%if cpuflag(avx512icl) + mova m5, [bilin_v_perm16] + movu xm0, [srcq+strideq*0] +.v_w16_loop: + movu xm2, [srcq+strideq*2] + vinserti128 ym1, ym0, [srcq+strideq*1], 1 + vpermt2b m1, m5, m2 + vinserti128 ym2, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + movu xm0, [srcq+strideq*0] + vpermt2b m2, m5, m0 + pmaddubsw m1, m6 + pmaddubsw m2, m6 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 +%else vbroadcasti128 m0, [srcq+strideq*0] .v_w16_loop: vbroadcasti128 m1, [srcq+strideq*2] vbroadcasti128 m2, [srcq+strideq*1] vbroadcasti128 m3, [srcq+stride3q ] lea srcq, [srcq+strideq*4] - shufpd m4, m0, m1, 0x0c ; 0 2 + shufpd m4, m0, m1, 0x0c ; 0 2 ; 0l2l 0h2h vbroadcasti128 m0, [srcq+strideq*0] - shufpd m2, m2, m3, 0x0c ; 1 3 - shufpd m1, m1, m0, 0x0c ; 2 4 + shufpd m2, m2, m3, 0x0c ; 1 3 ; 1l3l 1h3h + shufpd m1, m1, m0, 0x0c ; 2 4 ; 2l4l 2h4h punpcklbw m3, m2, m4 punpcklbw m5, m1, m2 punpckhbw m1, m2 punpckhbw m2, m4 pmaddubsw m3, m6 @@ -1103,34 +1389,58 @@ pmaddubsw m1, m6 mova [tmpq+32*0], m3 mova [tmpq+32*1], m5 mova [tmpq+32*2], m2 mova [tmpq+32*3], m1 +%endif add tmpq, 32*4 sub hd, 4 jg .v_w16_loop RET .v_w32: - vpermq m0, [srcq+strideq*0], q3120 +%if cpuflag(avx512icl) + mova m5, [bilin_v_perm32] + movu ym0, [srcq+strideq*0] .v_w32_loop: - vpermq m1, [srcq+strideq*1], q3120 - vpermq m2, [srcq+strideq*2], q3120 - vpermq m3, [srcq+stride3q ], q3120 + movu ym2, [srcq+strideq*1] + movu ym3, [srcq+strideq*2] + movu ym4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpermt2b m0, m5, m2 + vpermt2b m2, m5, m3 + vpermt2b m3, m5, m4 + pmaddubsw m1, m0, m6 + movu ym0, [srcq+strideq*0] + vpermt2b m4, m5, m0 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + pmaddubsw m4, m6 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + mova [tmpq+64*2], m3 + mova [tmpq+64*3], m4 + add tmpq, 64*4 +%else + vpermq ym0, [srcq+strideq*0], q3120 +.v_w32_loop: + vpermq ym1, [srcq+strideq*1], q3120 + vpermq ym2, [srcq+strideq*2], q3120 + vpermq ym3, [srcq+stride3q ], q3120 lea srcq, [srcq+strideq*4] punpcklbw m4, m1, m0 punpckhbw m5, m1, m0 - vpermq m0, [srcq+strideq*0], q3120 + vpermq ym0, [srcq+strideq*0], q3120 pmaddubsw m4, m6 pmaddubsw m5, m6 - mova [tmpq+32*0], m4 - mova [tmpq+32*1], m5 + mova [tmpq+32*0], ym4 + mova [tmpq+32*1], ym5 punpcklbw m4, m2, m1 punpckhbw m5, m2, m1 pmaddubsw m4, m6 pmaddubsw m5, m6 - mova [tmpq+32*2], m4 - mova [tmpq+32*3], m5 + mova [tmpq+32*2], ym4 + mova [tmpq+32*3], ym5 add tmpq, 32*8 punpcklbw m4, m3, m2 punpckhbw m5, m3, m2 punpcklbw m1, m0, m3 punpckhbw m2, m0, m3 @@ -1140,14 +1450,36 @@ pmaddubsw m2, m6 mova [tmpq-32*4], m4 mova [tmpq-32*3], m5 mova [tmpq-32*2], m1 mova [tmpq-32*1], m2 +%endif sub hd, 4 jg .v_w32_loop RET .v_w64: +%if cpuflag(avx512) + mova m5, [bilin_v_perm64] + vpermq m0, m5, [srcq+strideq*0] +.v_w64_loop: + vpermq m1, m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m1, m0 + punpckhbw m2, m1, m0 + vpermq m0, m5, [srcq+strideq*0] + punpcklbw m3, m0, m1 + punpckhbw m1, m0, m1 + pmaddubsw m4, m6 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + pmaddubsw m1, m6 + mova [tmpq+64*0], m4 + mova [tmpq+64*1], m2 + mova [tmpq+64*2], m3 + mova [tmpq+64*3], m1 + add tmpq, 64*4 +%else vpermq m0, [srcq+strideq*0+32*0], q3120 vpermq m1, [srcq+strideq*0+32*1], q3120 .v_w64_loop: vpermq m2, [srcq+strideq*1+32*0], q3120 vpermq m3, [srcq+strideq*1+32*1], q3120 @@ -1177,14 +1509,53 @@ pmaddubsw m3, m6 mova [tmpq-32*4], m4 mova [tmpq-32*3], m5 mova [tmpq-32*2], m2 mova [tmpq-32*1], m3 +%endif sub hd, 2 jg .v_w64_loop RET .v_w128: +%if cpuflag(avx512) + mova m5, [bilin_v_perm64] + vpermq m0, m5, [srcq+strideq*0+ 0] + vpermq m1, m5, [srcq+strideq*0+64] +.v_w128_loop: + vpermq m2, m5, [srcq+strideq*1+ 0] + vpermq m3, m5, [srcq+strideq*1+64] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m2, m0 + punpckhbw m0, m2, m0 + pmaddubsw m4, m6 + pmaddubsw m0, m6 + mova [tmpq+64*0], m4 + mova [tmpq+64*1], m0 + punpcklbw m4, m3, m1 + punpckhbw m1, m3, m1 + pmaddubsw m4, m6 + pmaddubsw m1, m6 + mova [tmpq+64*2], m4 + mova [tmpq+64*3], m1 + vpermq m0, m5, [srcq+strideq*0+ 0] + vpermq m1, m5, [srcq+strideq*0+64] + punpcklbw m4, m0, m2 + punpckhbw m2, m0, m2 + pmaddubsw m4, m6 + pmaddubsw m2, m6 + mova [tmpq+64*4], m4 + mova [tmpq+64*5], m2 + punpcklbw m4, m1, m3 + punpckhbw m3, m1, m3 + pmaddubsw m4, m6 + pmaddubsw m3, m6 + mova [tmpq+64*6], m4 + mova [tmpq+64*7], m3 + add tmpq, 64*8 + sub hd, 2 + jg .v_w128_loop +%else mov t0, tmpq mov t1, srcq lea t2d, [hq+(3<<8)] .v_w128_loop0: vpermq m0, [srcq+strideq*0], q3120 @@ -1205,61 +1576,87 @@ mova [tmpq+32*8], m4 mova [tmpq+32*9], m5 add tmpq, 32*16 sub hd, 2 jg .v_w128_loop - mov hb, t2b + movzx hd, t2b add t0, 64 add t1, 32 mov tmpq, t0 mov srcq, t1 sub t2d, 1<<8 jg .v_w128_loop0 +%endif RET .hv: ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) %assign stack_offset stack_offset - stack_size_padded WIN64_SPILL_XMM 7 movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)] shl mxyd, 11 +%if cpuflag(avx512) + vpbroadcastw m6, mxyd +%else movd xm6, mxyd + vpbroadcastw m6, xm6 +%endif add wq, t2 lea stride3q, [strideq*3] - vpbroadcastw m6, xm6 jmp wq .hv_w4: - vbroadcasti128 m4, [bilin_h_shuf4] - vpbroadcastq m0, [srcq+strideq*0] - pshufb m0, m4 - pmaddubsw m0, m5 + vbroadcasti128 ym4, [bilin_h_shuf4] + vpbroadcastq ym0, [srcq+strideq*0] + pshufb ym0, ym4 + pmaddubsw ym0, ym5 .hv_w4_loop: movq xm1, [srcq+strideq*1] movhps xm1, [srcq+strideq*2] movq xm2, [srcq+stride3q ] lea srcq, [srcq+strideq*4] movhps xm2, [srcq+strideq*0] - vinserti128 m1, m1, xm2, 1 + vinserti128 ym1, xm2, 1 + pshufb ym1, ym4 + pmaddubsw ym1, ym5 ; 1 2 3 4 +%if cpuflag(avx512) + valignq ym2, ym1, ym0, 3 ; 0 1 2 3 +%else + vpblendd ym2, ym1, ym0, 0xc0 + vpermq ym2, ym2, q2103 ; 0 1 2 3 +%endif + mova ym0, ym1 + psubw ym1, ym2 + pmulhrsw ym1, ym6 + paddw ym1, ym2 + mova [tmpq], ym1 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: +%if cpuflag(avx512) + vbroadcasti128 m4, [bilin_h_shuf8] +%endif + vbroadcasti128 m0, [srcq+strideq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu xm1, [srcq+strideq*1] +%if cpuflag(avx512) + vinserti128 ym1, [srcq+strideq*2], 1 + vinserti128 m1, [srcq+stride3q ], 2 + lea srcq, [srcq+strideq*4] + vinserti128 m1, [srcq+strideq*0], 3 pshufb m1, m4 pmaddubsw m1, m5 ; 1 2 3 4 - vpblendd m2, m1, m0, 0xc0 - vpermq m2, m2, q2103 ; 0 1 2 3 + valignq m2, m1, m0, 6 ; 0 1 2 3 mova m0, m1 psubw m1, m2 pmulhrsw m1, m6 paddw m1, m2 mova [tmpq], m1 - add tmpq, 32 - sub hd, 4 - jg .hv_w4_loop - RET -.hv_w8: - vbroadcasti128 m0, [srcq+strideq*0] - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w8_loop: - movu xm1, [srcq+strideq*1] +%else vinserti128 m1, m1, [srcq+strideq*2], 1 movu xm2, [srcq+stride3q ] lea srcq, [srcq+strideq*4] vinserti128 m2, m2, [srcq+strideq*0], 1 pshufb m1, m4 @@ -1274,25 +1671,53 @@ psubw m3, m0, m2 pmulhrsw m3, m6 paddw m3, m2 mova [tmpq+32*0], m1 mova [tmpq+32*1], m3 +%endif add tmpq, 32*2 sub hd, 4 jg .hv_w8_loop RET .hv_w16: - movu m0, [srcq+strideq*0+8*0] - vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm16] + vbroadcasti32x8 m0, [srcq+strideq*0] + vpermb m0, m4, m0 +%else + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 pshufb m0, m4 +%endif pmaddubsw m0, m5 .hv_w16_loop: - movu xm1, [srcq+strideq*1+8*0] - vinserti128 m1, m1, [srcq+strideq*1+8*1], 1 - lea srcq, [srcq+strideq*2] - movu xm2, [srcq+strideq*0+8*0] - vinserti128 m2, m2, [srcq+strideq*0+8*1], 1 +%if cpuflag(avx512icl) + movu ym1, [srcq+strideq*1] + vinserti32x8 m1, [srcq+strideq*2], 1 + movu ym2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti32x8 m2, [srcq+strideq*0], 1 + vpermb m1, m4, m1 + vpermb m2, m4, m2 + pmaddubsw m1, m5 ; 1 2 + vshufi32x4 m3, m0, m1, q1032 ; 0 1 + pmaddubsw m0, m2, m5 ; 3 4 + vshufi32x4 m2, m1, m0, q1032 ; 2 3 + psubw m1, m3 + pmulhrsw m1, m6 + paddw m1, m3 + psubw m3, m0, m2 + pmulhrsw m3, m6 + paddw m3, m2 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m3 +%else + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + movu xm2, [srcq+strideq*0+8*0] + vinserti128 m2, [srcq+strideq*0+8*1], 1 pshufb m1, m4 pshufb m2, m4 pmaddubsw m1, m5 psubw m3, m1, m0 pmulhrsw m3, m6 @@ -1301,19 +1726,41 @@ psubw m2, m0, m1 pmulhrsw m2, m6 paddw m2, m1 mova [tmpq+32*0], m3 mova [tmpq+32*1], m2 - add tmpq, 32*2 - sub hd, 2 +%endif + add tmpq, mmsize*2 + sub hd, mmsize*2/(16*2) jg .hv_w16_loop RET .hv_w32: - movu m0, [srcq+8*0] - vinserti128 m0, m0, [srcq+8*1], 1 - movu m1, [srcq+8*2] - vinserti128 m1, m1, [srcq+8*3], 1 +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+strideq*0] + pmaddubsw m0, m5 +.hv_w32_loop: + vpermb m1, m4, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermb m2, m4, [srcq+strideq*0] + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+64*0], m3 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 2 +%else + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 pshufb m0, m4 pshufb m1, m4 pmaddubsw m0, m5 pmaddubsw m1, m5 .hv_w32_loop: @@ -1336,18 +1783,45 @@ paddw m3, m1 mova m1, m2 mova [tmpq+32], m3 add tmpq, 32*2 dec hd +%endif jg .hv_w32_loop RET .hv_w64: +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w64_loop: + add srcq, strideq + vpermb m2, m4, [srcq+32*0] + vpermb m3, m4, [srcq+32*1] + pmaddubsw m2, m5 + pmaddubsw m3, m5 + psubw m7, m2, m0 + psubw m8, m3, m1 + pmulhrsw m7, m6 + pmulhrsw m8, m6 + paddw m7, m0 + paddw m8, m1 + mova [tmpq+ 0], m7 + mova [tmpq+64], m8 + mova m0, m2 + mova m1, m3 + add tmpq, 64*2 + dec hd + jg .hv_w64_loop +%else mov t0, tmpq mov t1, srcq lea t2d, [hq+(3<<8)] .hv_w64_loop0: - movu m0, [srcq+strideq*0+8*0] + movu xm0, [srcq+strideq*0+8*0] vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 pshufb m0, m4 pmaddubsw m0, m5 .hv_w64_loop: movu xm1, [srcq+strideq*1+8*0] @@ -1368,24 +1842,69 @@ mova [tmpq+32*0], m3 add tmpq, 32*8 mova [tmpq-32*4], m2 sub hd, 2 jg .hv_w64_loop - mov hb, t2b + movzx hd, t2b add t0, 32 add t1, 16 mov tmpq, t0 mov srcq, t1 sub t2d, 1<<8 jg .hv_w64_loop0 +%endif RET .hv_w128: +%if cpuflag(avx512icl) + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + vpermb m2, m4, [srcq+32*2] + vpermb m3, m4, [srcq+32*3] + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 +.hv_w128_loop: + add srcq, strideq + vpermb m7, m4, [srcq+32*0] + vpermb m8, m4, [srcq+32*1] + vpermb m9, m4, [srcq+32*2] + vpermb m10, m4, [srcq+32*3] + pmaddubsw m7, m5 + pmaddubsw m8, m5 + pmaddubsw m9, m5 + pmaddubsw m10, m5 + psubw m11, m7, m0 + psubw m12, m8, m1 + psubw m13, m9, m2 + psubw m14, m10, m3 + pmulhrsw m11, m6 + pmulhrsw m12, m6 + pmulhrsw m13, m6 + pmulhrsw m14, m6 + paddw m11, m0 + paddw m12, m1 + paddw m13, m2 + paddw m14, m3 + mova [tmpq+64*0], m11 + mova [tmpq+64*1], m12 + mova [tmpq+64*2], m13 + mova [tmpq+64*3], m14 + mova m0, m7 + mova m1, m8 + mova m2, m9 + mova m3, m10 + add tmpq, 64*4 + dec hd + jg .hv_w128_loop +%else mov t0, tmpq mov t1, srcq lea t2d, [hq+(7<<8)] .hv_w128_loop0: - movu m0, [srcq+strideq*0+8*0] + movu xm0, [srcq+strideq*0+8*0] vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 pshufb m0, m4 pmaddubsw m0, m5 .hv_w128_loop: movu xm1, [srcq+strideq*1+8*0] @@ -1406,18 +1925,20 @@ mova [tmpq+32*0], m3 mova [tmpq+32*8], m2 add tmpq, 32*16 sub hd, 2 jg .hv_w128_loop - mov hb, t2b - add t0, 32 - add t1, 16 + movzx hd, t2b + add t0, mmsize + add t1, mmsize/2 mov tmpq, t0 mov srcq, t1 sub t2d, 1<<8 jg .hv_w128_loop0 +%endif RET +%endmacro ; int8_t subpel_filters[5][15][8] %assign FILTER_REGULAR (0*15 << 16) | 3*15 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 @@ -1601,12 +2122,12 @@ .v: %assign stack_offset stack_offset - stack_size_padded WIN64_SPILL_XMM 16 movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd tzcnt r6d, wd movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)] vpbroadcastd m7, [pw_512] lea myq, [r8+myq*8+subpel_filters-put_avx2] vpbroadcastw m8, [myq+0] @@ -1804,11 +2325,11 @@ mova [dstq+dsq*0], xm14 vextracti128 [dstq+dsq*1], m14, 1 lea dstq, [dstq+dsq*2] sub hd, 2 jg .v_w16_loop - mov hb, r6b + movzx hd, r6b add r4, 16 add r7, 16 mov dstq, r4 mov srcq, r7 sub r6d, 1<<8 @@ -1822,12 +2343,12 @@ movzx mxd, mxb dec srcq vpbroadcastd m7, [r8+mxq*8+subpel_filters-put_avx2+2] movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] lea ss3q, [ssq*3] sub srcq, ss3q punpcklbw m0, m0 psraw m0, 8 ; sign-extend @@ -1963,12 +2484,12 @@ sub srcq, 3 vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+0] vpbroadcastd m11, [r8+mxq*8+subpel_filters-put_avx2+4] movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] lea ss3q, [ssq*3] sub srcq, ss3q punpcklbw m0, m0 psraw m0, 8 ; sign-extend @@ -2073,33 +2594,117 @@ movq [dstq+dsq*0], xm7 movhps [dstq+dsq*1], xm7 lea dstq, [dstq+dsq*2] sub hd, 2 jg .hv_w8_loop - mov hb, r6b + movzx hd, r6b add r4, 8 add r7, 8 mov dstq, r4 mov srcq, r7 sub r6d, 1<<8 jg .hv_w8_loop0 RET -%if WIN64 -DECLARE_REG_TMP 6, 4 -%else -DECLARE_REG_TMP 6, 7 -%endif +%macro PREP_8TAP_H 0 + %if cpuflag(avx512) + vpermb m10, m5, m0 + vpermb m11, m5, m1 + vpermb m12, m6, m0 + vpermb m13, m6, m1 + vpermb m14, m7, m0 + vpermb m15, m7, m1 + mova m0, m4 + mova m2, m4 + mova m1, m4 + mova m3, m4 + vpdpbusd m0, m10, m8 + vpdpbusd m2, m12, m8 + vpdpbusd m1, m11, m8 + vpdpbusd m3, m13, m8 + vpdpbusd m0, m12, m9 + vpdpbusd m2, m14, m9 + vpdpbusd m1, m13, m9 + vpdpbusd m3, m15, m9 + packssdw m0, m2 + packssdw m1, m3 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+ 0], m0 + mova [tmpq+64], m1 + %else + pshufb m1, m0, m5 + pshufb m2, m0, m6 + pshufb m3, m0, m7 + pmaddubsw m1, m8 + pmaddubsw m0, m2, m8 + pmaddubsw m2, m9 + pmaddubsw m3, m9 + paddw m1, m2 + paddw m0, m3 + phaddw m0, m1, m0 + pmulhrsw m0, m4 + %endif +%endmacro + +%macro PREP_8TAP_V_W4 5 ; round, weights + movd xm0, [srcq+strideq*0] + vpbroadcastd ym1, [srcq+strideq*2] + vpbroadcastd xm2, [srcq+strideq*1] + vpbroadcastd ym3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd ym1, ym1, ym0, 0x01 ; 0 2 2 _ 2 _ _ _ + vpblendd ym3, ym3, ym2, 0x03 ; 1 1 3 3 3 3 _ _ + vpbroadcastd ym0, [srcq+strideq*0] + vpbroadcastd ym2, [srcq+strideq*1] + vpblendd ym1, ym1, ym0, 0x68 ; 0 2 2 4 2 4 4 _ + vpbroadcastd ym0, [srcq+strideq*2] + vbroadcasti128 ym5, [deint_shuf4] + vpblendd ym3, ym3, ym2, 0xc0 ; 1 1 3 3 3 3 5 5 + vpblendd ym2, ym3, ym1, 0x55 ; 0 1 2 3 2 3 4 5 + vpblendd ym3, ym3, ym1, 0xaa ; 1 2 3 4 3 4 5 _ + punpcklbw ym1, ym2, ym3 ; 01 12 23 34 + vpblendd ym3, ym3, ym0, 0x80 ; 1 2 3 4 3 4 5 6 + punpckhbw ym2, ym3 ; 23 34 45 56 +.v_w4_loop: + pinsrd xm0, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vpbroadcastd ym3, [srcq+strideq*0] + vpbroadcastd ym4, [srcq+strideq*1] + vpblendd ym3, ym3, ym4, 0x20 ; _ _ 8 _ 8 9 _ _ + vpblendd ym3, ym3, ym0, 0x03 ; 6 7 8 _ 8 9 _ _ + vpbroadcastd ym0, [srcq+strideq*2] + vpblendd ym3, ym3, ym0, 0x40 ; 6 7 8 _ 8 9 a _ + pshufb ym3, ym5 ; 67 78 89 9a + pmaddubsw ym4, ym1, ym%2 + vperm2i128 ym1, ym2, ym3, 0x21 ; 45 56 67 78 + pmaddubsw ym2, ym%3 + paddw ym4, ym2 + mova ym2, ym3 + pmaddubsw ym3, ym%5 + paddw ym3, ym4 + pmaddubsw ym4, ym1, ym%4 + paddw ym3, ym4 + pmulhrsw ym3, ym%1 + mova [tmpq], ym3 +%endmacro + %macro PREP_8TAP_FN 3 ; type, type_h, type_v cglobal prep_8tap_%1 mov t0d, FILTER_%2 mov t1d, FILTER_%3 %ifnidn %1, sharp_smooth ; skip the jump in the last filter jmp mangle(private_prefix %+ _prep_8tap %+ SUFFIX) %endif %endmacro +%macro PREP_8TAP 0 + %if WIN64 + DECLARE_REG_TMP 6, 4 + %else + DECLARE_REG_TMP 6, 7 + %endif PREP_8TAP_FN regular, REGULAR, REGULAR PREP_8TAP_FN regular_sharp, REGULAR, SHARP PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR PREP_8TAP_FN smooth, SMOOTH, SMOOTH @@ -2111,11 +2716,11 @@ cglobal prep_8tap, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h imul myd, mym, 0x010101 add myd, t1d ; 8tap_v, my, 4tap_v - lea r7, [prep_avx2] + lea r7, [prep%+SUFFIX] movsxd wq, wm movifnidn hd, hm test mxd, 0xf00 jnz .h test myd, 0xf00 @@ -2129,108 +2734,196 @@ %endif jmp wq .h: test myd, 0xf00 jnz .hv +%if cpuflag(avx512) + vpbroadcastd m4, [pd_2] +%else + vpbroadcastd m4, [pw_8192] vbroadcasti128 m5, [subpel_h_shufA] - vpbroadcastd m4, [pw_8192] +%endif WIN64_SPILL_XMM 10 cmp wd, 4 je .h_w4 tzcnt wd, wd +%if notcpuflag(avx512) vbroadcasti128 m6, [subpel_h_shufB] vbroadcasti128 m7, [subpel_h_shufC] +%endif shr mxd, 16 sub srcq, 3 movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] - vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx2+0] - vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep_avx2+4] + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] + vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] add wq, r7 jmp wq .h_w4: +%if cpuflag(avx512) + mov r3d, 0x4 + kmovb k1, r3d + vbroadcasti128 ym5, [subpel_h_shufA] +%endif movzx mxd, mxb dec srcq - vpbroadcastd m3, [r7+mxq*8+subpel_filters-prep_avx2+2] + vpbroadcastd ym6, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] lea stride3q, [strideq*3] .h_w4_loop: +%if cpuflag(avx512icl) + mova ym0, ym4 + mova ym1, ym4 + movq xm2, [srcq+strideq*0] + movq xm3, [srcq+strideq*1] + vpbroadcastq ym2{k1}, [srcq+strideq*2] + vpbroadcastq ym3{k1}, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pshufb ym2, ym5 + pshufb ym3, ym5 + vpdpbusd ym0, ym2, ym6 + vpdpbusd ym1, ym3, ym6 + packssdw ym0, ym1 + psraw ym0, 2 +%else movq xm0, [srcq+strideq*0] vpbroadcastq m2, [srcq+strideq*2] movq xm1, [srcq+strideq*1] vpblendd m0, m0, m2, 0xf0 vpbroadcastq m2, [srcq+stride3q ] lea srcq, [srcq+strideq*4] vpblendd m1, m1, m2, 0xf0 pshufb m0, m5 pshufb m1, m5 - pmaddubsw m0, m3 - pmaddubsw m1, m3 + pmaddubsw m0, m6 + pmaddubsw m1, m6 phaddw m0, m1 pmulhrsw m0, m4 - mova [tmpq], m0 +%endif + mova [tmpq], ym0 add tmpq, 32 sub hd, 4 jg .h_w4_loop RET .h_w8: -%macro PREP_8TAP_H 0 - pshufb m1, m0, m6 - pshufb m2, m0, m7 - pshufb m0, m5 - pmaddubsw m3, m1, m8 - pmaddubsw m1, m9 - pmaddubsw m2, m9 - pmaddubsw m0, m8 - paddw m2, m3 - paddw m0, m1 - phaddw m0, m2 - pmulhrsw m0, m4 -%endmacro - movu xm0, [srcq+strideq*0] - vinserti128 m0, m0, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] +%if cpuflag(avx512) + vbroadcasti128 m5, [subpel_h_shufA] + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + lea stride3q, [strideq*3] +%endif +.h_w8_loop: + movu xm0, [srcq+strideq*0] + vinserti128 ym0, [srcq+strideq*1], 1 +%if cpuflag(avx512) + vinserti128 m0, [srcq+strideq*2], 2 + vinserti128 m0, [srcq+stride3q ], 3 +%endif + lea srcq, [srcq+strideq*(mmsize/(8*2))] +%if cpuflag(avx512icl) + mova m10, m4 + mova m11, m4 + pshufb m1, m0, m5 + pshufb m2, m0, m6 + pshufb m3, m0, m7 + vpdpbusd m10, m1, m8 + vpdpbusd m11, m2, m8 + vpdpbusd m10, m2, m9 + vpdpbusd m11, m3, m9 + packssdw m10, m11 + psraw m0, m10, 2 +%else PREP_8TAP_H +%endif mova [tmpq], m0 - add tmpq, 32 - sub hd, 2 - jg .h_w8 + add tmpq, mmsize + sub hd, mmsize/(8*2) + jg .h_w8_loop RET .h_w16: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 +%if cpuflag(avx512icl) + mova m5, [spel_h_perm16a] + mova m6, [spel_h_perm16b] + mova m7, [spel_h_perm16c] + lea stride3q, [strideq*3] +.h_w16_loop: + movu ym0, [srcq+strideq*0] + movu ym1, [srcq+strideq*2] + vinserti32x8 m0, [srcq+strideq*1], 1 + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + PREP_8TAP_H +%else +.h_w16_loop: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 PREP_8TAP_H mova [tmpq+32*0], m0 movu xm0, [srcq+strideq*1+8*0] vinserti128 m0, m0, [srcq+strideq*1+8*1], 1 lea srcq, [srcq+strideq*2] PREP_8TAP_H mova [tmpq+32*1], m0 - add tmpq, 64 - sub hd, 2 - jg .h_w16 +%endif + add tmpq, mmsize*2 + sub hd, mmsize*2/(16*2) + jg .h_w16_loop RET .h_w32: +%if cpuflag(avx512icl) + mova m5, [spel_h_perm32a] + mova m6, [spel_h_perm32b] + mova m7, [spel_h_perm32c] +.h_w32_loop: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + add tmpq, 64*2 + sub hd, 2 + jg .h_w32_loop + RET +%else xor r6d, r6d jmp .h_start +%endif .h_w64: +%if cpuflag(avx512) + xor r6d, r6d +%else mov r6, -32*1 +%endif jmp .h_start .h_w128: +%if cpuflag(avx512) + mov r6, -64*1 +%else mov r6, -32*3 +%endif .h_start: +%if cpuflag(avx512) + mova m5, [spel_h_perm32a] + mova m6, [spel_h_perm32b] + mova m7, [spel_h_perm32c] +%endif sub srcq, r6 mov r5, r6 .h_loop: - movu xm0, [srcq+r6+8*0] - vinserti128 m0, m0, [srcq+r6+8*1], 1 +%if cpuflag(avx512icl) + movu m0, [srcq+r6+32*0] + movu m1, [srcq+r6+32*1] + PREP_8TAP_H +%else + movu xm0, [srcq+r6+8*0] + vinserti128 ym0, [srcq+r6+8*1], 1 PREP_8TAP_H mova [tmpq+32*0], m0 - movu xm0, [srcq+r6+8*2] - vinserti128 m0, m0, [srcq+r6+8*3], 1 + movu xm0, [srcq+r6+8*2] + vinserti128 ym0, [srcq+r6+8*3], 1 PREP_8TAP_H mova [tmpq+32*1], m0 - add tmpq, 64 - add r6, 32 +%endif + add tmpq, mmsize*2 + add r6, mmsize jle .h_loop add srcq, strideq mov r6, r5 dec hd jg .h_loop @@ -2241,66 +2934,93 @@ movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. shr myd, 16 ; Note that the code is 8-tap only, having cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 cmove myd, mxd ; had a negligible effect on performance. ; TODO: Would a 6-tap code path be worth it? +%if cpuflag(avx512) + tzcnt wd, wd + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_v)] + add wq, r7 +%endif + lea myq, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q vpbroadcastd m7, [pw_8192] - lea myq, [r7+myq*8+subpel_filters-prep_avx2] vpbroadcastw m8, [myq+0] vpbroadcastw m9, [myq+2] vpbroadcastw m10, [myq+4] vpbroadcastw m11, [myq+6] - lea stride3q, [strideq*3] - sub srcq, stride3q +%if cpuflag(avx512) + jmp wq +%else cmp wd, 8 jg .v_w16 je .v_w8 +%endif .v_w4: - movd xm0, [srcq+strideq*0] - vpbroadcastd m1, [srcq+strideq*2] - vpbroadcastd xm2, [srcq+strideq*1] - vpbroadcastd m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpblendd m1, m1, m0, 0x01 ; 0 2 2 _ 2 _ _ _ - vpblendd m3, m3, m2, 0x03 ; 1 1 3 3 3 3 _ _ - vpbroadcastd m0, [srcq+strideq*0] - vpbroadcastd m2, [srcq+strideq*1] - vpblendd m1, m1, m0, 0x68 ; 0 2 2 4 2 4 4 _ - vpbroadcastd m0, [srcq+strideq*2] - vbroadcasti128 m6, [deint_shuf4] - vpblendd m3, m3, m2, 0xc0 ; 1 1 3 3 3 3 5 5 - vpblendd m2, m3, m1, 0x55 ; 0 1 2 3 2 3 4 5 - vpblendd m3, m3, m1, 0xaa ; 1 2 3 4 3 4 5 _ - punpcklbw m1, m2, m3 ; 01 12 23 34 - vpblendd m3, m3, m0, 0x80 ; 1 2 3 4 3 4 5 6 - punpckhbw m2, m3 ; 23 34 45 56 -.v_w4_loop: - pinsrd xm0, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - vpbroadcastd m3, [srcq+strideq*0] - vpbroadcastd m4, [srcq+strideq*1] - vpblendd m3, m3, m4, 0x20 ; _ _ 8 _ 8 9 _ _ - vpblendd m3, m3, m0, 0x03 ; 6 7 8 _ 8 9 _ _ - vpbroadcastd m0, [srcq+strideq*2] - vpblendd m3, m3, m0, 0x40 ; 6 7 8 _ 8 9 a _ - pshufb m3, m6 ; 67 78 89 9a - pmaddubsw m4, m1, m8 - vperm2i128 m1, m2, m3, 0x21 ; 45 56 67 78 - pmaddubsw m2, m9 - paddw m4, m2 - mova m2, m3 - pmaddubsw m3, m11 - paddw m3, m4 - pmaddubsw m4, m1, m10 - paddw m3, m4 - pmulhrsw m3, m7 - mova [tmpq], m3 +%if cpuflag(avx512) + AVX512_MM_PERMUTATION + PREP_8TAP_V_W4 23, 24, 25, 26, 27 + AVX512_MM_PERMUTATION +%else + PREP_8TAP_V_W4 7, 8, 9, 10, 11 +%endif add tmpq, 32 sub hd, 4 jg .v_w4_loop +%if cpuflag(avx512) + vzeroupper +%endif RET .v_w8: +%if cpuflag(avx512) + mov r3d, 0xf044 + kmovw k1, r3d + kshiftrw k2, k1, 8 + movq xm0, [srcq+strideq*0] + vpbroadcastq ym1, [srcq+strideq*1] + vpbroadcastq m2, [srcq+strideq*2] + vpbroadcastq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m4, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + vpbroadcastq m6, [srcq+strideq*2] + vmovdqa64 ym0{k1}, ym1 + vmovdqa64 ym1{k1}, ym2 + vmovdqa64 m2{k1}, m3 + vmovdqa64 m3{k1}, m4 + vmovdqa64 m4{k1}, m5 + vmovdqa64 m5{k1}, m6 + punpcklbw ym0, ym1 ; 01 12 __ __ + punpcklbw m2, m3 ; 23 34 23 34 + punpcklbw m4, m5 ; 45 56 45 56 + vmovdqa64 m0{k2}, m2 ; 01 12 23 34 + vmovdqa64 m2{k2}, m4 ; 23 34 45 56 +.v_w8_loop: + vpbroadcastq m1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + pmaddubsw m14, m0, m8 + pmaddubsw m15, m2, m9 + vpblendmq m0{k1}, m6, m1 + vpblendmq m2{k1}, m1, m3 + vpbroadcastq m6, [srcq+strideq*2] + paddw m14, m15 + punpcklbw m2, m0, m2 ; 67 78 67 78 + vpblendmq m12{k1}, m3, m5 + vpblendmq m13{k1}, m5, m6 + vpblendmq m0{k2}, m4, m2 ; 45 56 67 78 + punpcklbw m4, m12, m13 ; 89 9a 89 9a + vmovdqa64 m2{k2}, m4 ; 67 78 89 9a + pmaddubsw m12, m0, m10 + pmaddubsw m13, m2, m11 + paddw m14, m12 + paddw m14, m13 + pmulhrsw m14, m7 + mova [tmpq], m14 +%else movq xm1, [srcq+strideq*0] vpbroadcastq m4, [srcq+strideq*1] vpbroadcastq m2, [srcq+strideq*2] vpbroadcastq m5, [srcq+stride3q ] lea srcq, [srcq+strideq*4] @@ -2345,15 +3065,77 @@ paddw m6, m4 pmulhrsw m5, m7 pmulhrsw m6, m7 mova [tmpq+32*0], m5 mova [tmpq+32*1], m6 +%endif add tmpq, 32*2 sub hd, 4 jg .v_w8_loop RET .v_w16: +%if cpuflag(avx512) + mov r3d, 0xf0 + kmovb k1, r3d + vbroadcasti128 m0, [srcq+strideq*0] + vbroadcasti128 m1, [srcq+strideq*1] + vbroadcasti128 m2, [srcq+strideq*2] + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vbroadcasti128 m4, [srcq+strideq*0] + vbroadcasti128 m5, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*2] + vmovdqa64 m0{k1}, m1 + vmovdqa64 m1{k1}, m2 + vmovdqa64 m2{k1}, m3 + vmovdqa64 m3{k1}, m4 + vmovdqa64 m4{k1}, m5 + vmovdqa64 m5{k1}, m6 + shufpd m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b + shufpd m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b + shufpd m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_-- + shufpd m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_-- + punpckhbw m2, m0, m1 ; 23a 23b 34a 34b + punpcklbw m0, m1 ; 01a 01b 12a 12b + punpcklbw m4, m5 ; 45a 45b 56a 56b +.v_w16_loop: + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vbroadcasti128 m5, [srcq+strideq*0] + vpblendmq m1{k1}, m6, m3 + vmovdqa64 m3{k1}, m5 + pmaddubsw m12, m0, m8 + pmaddubsw m13, m2, m8 + pmaddubsw m14, m2, m9 + pmaddubsw m15, m4, m9 + pmaddubsw m0, m4, m10 + vbroadcasti128 m2, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*2] + paddw m12, m14 + paddw m13, m15 + paddw m12, m0 + vmovdqa64 m5{k1}, m2 + vmovdqa64 m2{k1}, m6 + mova m0, m4 + shufpd m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b + shufpd m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab + punpcklbw m2, m1, m3 ; 67a 67b 78a 78b + punpckhbw m4, m1, m3 ; 89a 89b 9Aa 9Ab + pmaddubsw m14, m2, m10 + pmaddubsw m15, m2, m11 + paddw m13, m14 + paddw m12, m15 + pmaddubsw m14, m4, m11 + paddw m13, m14 + pmulhrsw m12, m7 + pmulhrsw m13, m7 + mova [tmpq+ 0], m12 + mova [tmpq+64], m13 + add tmpq, 64*2 + sub hd, 4 + jg .v_w16_loop +%else lea r6d, [wq-16] mov r5, tmpq mov r7, srcq shl r6d, 4 mov r6b, hb @@ -2409,43 +3191,298 @@ mova [tmpq+wq*0], m14 mova [tmpq+wq*2], m15 lea tmpq, [tmpq+wq*4] sub hd, 2 jg .v_w16_loop - mov hb, r6b + movzx hd, r6b add r5, 32 add r7, 16 mov tmpq, r5 mov srcq, r7 sub r6d, 1<<8 jg .v_w16_loop0 +%endif + RET +%if cpuflag(avx512) +.v_w32: + mova m18, [bilin_v_perm64] + movu ym0, [srcq+strideq*0] + movu ym1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym2, [srcq+strideq*0] + movu ym3, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym4, [srcq+strideq*0] + movu ym5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym6, [srcq+strideq*0] + vpermq m0, m18, m0 + vpermq m1, m18, m1 + vpermq m2, m18, m2 + vpermq m3, m18, m3 + vpermq m4, m18, m4 + vpermq m5, m18, m5 + vpermq m6, m18, m6 + punpcklbw m0, m1 + punpcklbw m1, m2 + punpcklbw m2, m3 + punpcklbw m3, m4 + punpcklbw m4, m5 + punpcklbw m5, m6 +.v_w32_loop: + movu ym12, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym13, [srcq+strideq*0] + pmaddubsw m14, m0, m8 + pmaddubsw m16, m2, m9 + pmaddubsw m15, m1, m8 + pmaddubsw m17, m3, m9 + mova m0, m2 + mova m1, m3 + vpermq m12, m18, m12 + vpermq m13, m18, m13 + paddw m14, m16 + paddw m15, m17 + pmaddubsw m16, m4, m10 + pmaddubsw m17, m5, m10 + punpcklbw m6, m12 + punpcklbw m12, m13 + mova m2, m4 + mova m3, m5 + paddw m14, m16 + paddw m15, m17 + pmaddubsw m16, m6, m11 + pmaddubsw m17, m12, m11 + mova m4, m6 + mova m5, m12 + paddw m14, m16 + paddw m15, m17 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + mova m6, m13 + mova [tmpq+ 0], m14 + mova [tmpq+64], m15 + add tmpq, 64*2 + sub hd, 2 + jg .v_w32_loop + vzeroupper + RET +.v_w64: + mov r6d, hd + mov wd, 64 + jmp .v_start +.v_w128: + lea r6d, [(1<<8)+hq] + mov wd, 128 +.v_start: + WIN64_SPILL_XMM 27 + mova m26, [bilin_v_perm64] + mov r5, tmpq + mov r7, srcq +.v_loop0: + vpermq m0, m26, [srcq+strideq*0] + vpermq m1, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m2, m26, [srcq+strideq*0] + vpermq m3, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m4, m26, [srcq+strideq*0] + vpermq m5, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m6, m26, [srcq+strideq*0] + punpckhbw m12, m0, m1 + punpcklbw m0, m1 + punpckhbw m13, m1, m2 + punpcklbw m1, m2 + punpckhbw m14, m2, m3 + punpcklbw m2, m3 + punpckhbw m15, m3, m4 + punpcklbw m3, m4 + punpckhbw m16, m4, m5 + punpcklbw m4, m5 + punpckhbw m17, m5, m6 + punpcklbw m5, m6 +.v_loop: + vpermq m18, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m19, m26, [srcq+strideq*0] + pmaddubsw m20, m0, m8 + pmaddubsw m21, m12, m8 + pmaddubsw m22, m1, m8 + pmaddubsw m23, m13, m8 + mova m0, m2 + mova m12, m14 + mova m1, m3 + mova m13, m15 + pmaddubsw m2, m9 + pmaddubsw m14, m9 + pmaddubsw m3, m9 + pmaddubsw m15, m9 + punpckhbw m24, m6, m18 + punpcklbw m6, m18 + paddw m20, m2 + paddw m21, m14 + paddw m22, m3 + paddw m23, m15 + mova m2, m4 + mova m14, m16 + mova m3, m5 + mova m15, m17 + pmaddubsw m4, m10 + pmaddubsw m16, m10 + pmaddubsw m5, m10 + pmaddubsw m17, m10 + punpckhbw m25, m18, m19 + punpcklbw m18, m19 + paddw m20, m4 + paddw m21, m16 + paddw m22, m5 + paddw m23, m17 + mova m4, m6 + mova m16, m24 + mova m5, m18 + mova m17, m25 + pmaddubsw m6, m11 + pmaddubsw m24, m11 + pmaddubsw m18, m11 + pmaddubsw m25, m11 + paddw m20, m6 + paddw m21, m24 + paddw m22, m18 + paddw m23, m25 + pmulhrsw m20, m7 + pmulhrsw m21, m7 + pmulhrsw m22, m7 + pmulhrsw m23, m7 + mova m6, m19 + mova [tmpq+wq*0+ 0], m20 + mova [tmpq+wq*0+64], m21 + mova [tmpq+wq*2+ 0], m22 + mova [tmpq+wq*2+64], m23 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_loop + movzx hd, r6b + add r5, 64*2 + add r7, 64 + mov tmpq, r5 + mov srcq, r7 + sub r6d, 1<<8 + jg .v_loop0 +%endif RET .hv: %assign stack_offset stack_offset - stack_size_padded + %assign stack_size_padded 0 WIN64_SPILL_XMM 16 cmp wd, 4 - jg .hv_w8 - movzx mxd, mxb - dec srcq - mova m7, [subpel_h_shuf4] - vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx2+2] - pmovzxbd m9, [deint_shuf4] + je .hv_w4 + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] + vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] movzx mxd, myb shr myd, 16 cmp hd, 4 cmove myd, mxd - vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx2] +%if cpuflag(avx512) + tzcnt wd, wd + vpbroadcastd m8, [pd_2] + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_hv)] + vpbroadcastd m9, [pd_32] + add wq, r7 +%endif + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] lea stride3q, [strideq*3] sub srcq, stride3q punpcklbw m0, m0 psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 +%if cpuflag(avx512) + jmp wq +%else + jmp .hv_w8 +%endif +.hv_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q +%if cpuflag(avx512) + mov r3d, 0x04 + kmovb k1, r3d + kshiftlb k2, k1, 2 + kshiftlb k3, k1, 4 + vpbroadcastd m10, [pd_2] + vbroadcasti128 m16, [subpel_h_shufA] +%else + mova m7, [subpel_h_shuf4] + pmovzxbd m9, [deint_shuf4] vpbroadcastd m10, [pw_8192] +%endif + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend vpbroadcastd m11, [pd_32] pshufd m12, m0, q0000 pshufd m13, m0, q1111 pshufd m14, m0, q2222 pshufd m15, m0, q3333 +%if cpuflag(avx512icl) + movq xm3, [srcq+strideq*0] + vpbroadcastq ym2, [srcq+strideq*1] + vpbroadcastq ym3{k1}, [srcq+strideq*2] + vpbroadcastq m2{k2}, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3{k2}, [srcq+strideq*0] + vpbroadcastq m2{k3}, [srcq+strideq*1] + vpbroadcastq m3{k3}, [srcq+strideq*2] + mova m17, [spel_hv_perm4a] + movu m18, [spel_hv_perm4b] + mova m0, m10 + mova m1, m10 + pshufb m2, m16 + pshufb m3, m16 + vpdpbusd m0, m2, m8 + vpdpbusd m1, m3, m8 + packssdw m0, m1 ; _ 0 1 2 3 4 5 6 + psraw m0, 2 + vpermb m1, m17, m0 ; 01 12 23 34 + vpermb m2, m18, m0 ; 23 34 45 56 +.hv_w4_loop: + movq xm3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movq xm4, [srcq+strideq*0] + vpbroadcastq ym3{k1}, [srcq+strideq*1] + vpbroadcastq ym4{k1}, [srcq+strideq*2] + mova ym5, ym10 + mova ym6, ym10 + pshufb ym3, ym16 + pshufb ym4, ym16 + vpdpbusd ym5, ym3, ym8 + vpdpbusd ym6, ym4, ym8 + mova m7, m11 + packssdw ym5, ym6 ; 7 8 9 a _ _ _ _ + psraw ym5, 2 + valignq m0, m5, m0, 4 ; _ 4 5 6 7 8 9 a + vpdpwssd m7, m1, m12 + vpdpwssd m7, m2, m13 + vpermb m1, m17, m0 ; 45 56 67 78 + vpermb m2, m18, m0 ; 67 78 89 9a + vpdpwssd m7, m1, m14 + vpdpwssd m7, m2, m15 + psrad m7, 6 + vpmovdw [tmpq], m7 +%else vpbroadcastq m2, [srcq+strideq*0] vpbroadcastq m4, [srcq+strideq*1] vpbroadcastq m0, [srcq+strideq*2] vpbroadcastq m5, [srcq+stride3q ] lea srcq, [srcq+strideq*4] @@ -2453,23 +3490,23 @@ vpbroadcastq m6, [srcq+strideq*1] vpbroadcastq m1, [srcq+strideq*2] vpblendd m2, m2, m4, 0xcc ; 0 1 vpblendd m0, m0, m5, 0xcc ; 2 3 vpblendd m3, m3, m6, 0xcc ; 4 5 - pshufb m2, m7 - pshufb m0, m7 - pshufb m3, m7 - pshufb m1, m7 + pshufb m2, m7 ; 00 01 10 11 02 03 12 13 + pshufb m0, m7 ; 20 21 30 31 22 23 32 33 + pshufb m3, m7 ; 40 41 50 51 42 43 52 53 + pshufb m1, m7 ; 60 61 60 61 62 63 62 63 pmaddubsw m2, m8 pmaddubsw m0, m8 pmaddubsw m3, m8 pmaddubsw m1, m8 - phaddw m2, m0 - phaddw m3, m1 + phaddw m2, m0 ; 0a 1a 2a 3a 0b 1b 2b 3b + phaddw m3, m1 ; 4a 5a 6a __ 4b 5b 6b __ pmulhrsw m2, m10 pmulhrsw m3, m10 - palignr m4, m3, m2, 4 + palignr m4, m3, m2, 4 ; 1a 2a 3a 4a 1b 2b 3b 4b punpcklwd m1, m2, m4 ; 01 12 punpckhwd m2, m4 ; 23 34 pshufd m0, m3, q2121 punpcklwd m3, m0 ; 45 56 .hv_w4_loop: @@ -2510,32 +3547,104 @@ psrad m5, 6 psrad m6, 6 packssdw m5, m6 vpermd m5, m9, m5 mova [tmpq], m5 +%endif add tmpq, 32 sub hd, 4 jg .hv_w4_loop +%if cpuflag(avx512) + vzeroupper +%endif RET .hv_w8: - shr mxd, 16 - sub srcq, 3 - vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep_avx2+0] - vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep_avx2+4] - movzx mxd, myb - shr myd, 16 - cmp hd, 4 - cmove myd, mxd - vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx2] - lea stride3q, [strideq*3] - sub srcq, stride3q - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - pshufd m12, m0, q0000 - pshufd m13, m0, q1111 - pshufd m14, m0, q2222 - pshufd m15, m0, q3333 +%if cpuflag(avx512icl) + WIN64_SPILL_XMM 24 + vbroadcasti128 m16, [subpel_h_shufA] + vbroadcasti128 m17, [subpel_h_shufB] + vbroadcasti128 m18, [subpel_h_shufC] + vinserti128 ym0, [srcq+strideq*0], 1 + vinserti128 m0, [srcq+strideq*1], 2 + vinserti128 m0, [srcq+strideq*2], 3 + movu xm1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 ym1, [srcq+strideq*0], 1 + vinserti128 m1, [srcq+strideq*1], 2 + vinserti128 m1, [srcq+strideq*2], 3 + mova m2, m8 + mova m4, m8 + mova m3, m8 + mova m5, m8 + pshufb m20, m0, m16 + pshufb m21, m0, m17 + pshufb m22, m0, m18 + pshufb m23, m1, m16 + pshufb m6, m1, m17 + pshufb m7, m1, m18 + vpdpbusd m2, m20, m10 + vpdpbusd m4, m21, m10 + vpdpbusd m2, m21, m11 + vpdpbusd m4, m22, m11 + vpdpbusd m3, m23, m10 + vpdpbusd m5, m6, m10 + vpdpbusd m3, m6, m11 + vpdpbusd m5, m7, m11 + packssdw m2, m4 + packssdw m3, m5 + psraw m2, 2 ; _ 0 1 2 + psraw m3, 2 ; 3 4 5 6 + valignq m0, m3, m2, 2 ; 0 1 2 3 + valignq m1, m3, m2, 4 ; 1 2 3 4 + valignq m2, m3, m2, 6 ; 2 3 4 5 + punpcklwd m4, m0, m1 ; 01a 12a 23a 34a + punpckhwd m5, m0, m1 ; 01b 12b 23b 34b + punpcklwd m6, m2, m3 ; 23a 34a 45a 56a + punpckhwd m7, m2, m3 ; 23b 34b 45b 56b +.hv_w8_loop: + movu xm19, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 ym19, [srcq+strideq*0], 1 + vinserti128 m19, [srcq+strideq*1], 2 + vinserti128 m19, [srcq+strideq*2], 3 + mova m20, m9 + mova m21, m9 + mova m22, m8 + mova m23, m8 + vpdpwssd m20, m4, m12 + vpdpwssd m21, m5, m12 + vpdpwssd m20, m6, m13 + vpdpwssd m21, m7, m13 + pshufb m0, m19, m16 + pshufb m1, m19, m17 + pshufb m2, m19, m18 + vpdpbusd m22, m0, m10 + vpdpbusd m23, m1, m10 + vpdpbusd m22, m1, m11 + vpdpbusd m23, m2, m11 + packssdw m22, m23 + psraw m22, 2 ; 7 8 9 A + valignq m0, m22, m3, 2 ; 4 5 6 7 + valignq m1, m22, m3, 4 ; 5 6 7 8 + valignq m2, m22, m3, 6 ; 6 7 8 9 + mova m3, m22 + punpcklwd m4, m0, m1 ; 45a 56a 67a 78a + punpckhwd m5, m0, m1 ; 45b 56b 67b 78b + punpcklwd m6, m2, m3 ; 67a 78a 89a 9Aa + punpckhwd m7, m2, m3 ; 67b 78b 89b 9Ab + vpdpwssd m20, m4, m14 + vpdpwssd m21, m5, m14 + vpdpwssd m20, m6, m15 + vpdpwssd m21, m7, m15 + psrad m20, 6 + psrad m21, 6 + packssdw m20, m21 + mova [tmpq], m20 + add tmpq, 64 + sub hd, 4 + jg .hv_w8_loop +%else lea r6d, [wq-8] mov r5, tmpq mov r7, srcq shl r6d, 5 mov r6b, hb @@ -2617,18 +3726,154 @@ mova [tmpq+wq*0], xm7 vextracti128 [tmpq+wq*2], m7, 1 lea tmpq, [tmpq+wq*4] sub hd, 2 jg .hv_w8_loop - mov hb, r6b + movzx hd, r6b add r5, 16 add r7, 8 mov tmpq, r5 mov srcq, r7 sub r6d, 1<<8 jg .hv_w8_loop0 +%endif RET +%if cpuflag(avx512icl) +.hv_w16: + mov wd, 16*2 + jmp .hv_start +.hv_w32: + mov wd, 32*2 + jmp .hv_start +.hv_w64: + mov wd, 64*2 + jmp .hv_start +.hv_w128: + mov wd, 128*2 +.hv_start: + WIN64_SPILL_XMM 31 + mova m16, [spel_h_perm16a] + mova m17, [spel_h_perm16b] + mova m18, [spel_h_perm16c] + lea r6d, [wq*8-16*2*8+hq] + mov r5, tmpq + mov r7, srcq +.hv_loop0: + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym1, [srcq+strideq*0] + vinserti32x8 m1, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym2, [srcq+strideq*0] + vinserti32x8 m2, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym3, [srcq+strideq*0] + mova m4, m8 + mova m5, m8 + mova m6, m8 + mova m7, m8 + vpermb m19, m16, m0 + vpermb m20, m17, m0 + vpermb m21, m18, m0 + vpermb m22, m16, m1 + vpermb m23, m17, m1 + vpermb m24, m18, m1 + vpermb m25, m16, m2 + vpermb m26, m17, m2 + vpermb m27, m18, m2 + vpermb ym28, ym16, ym3 + vpermb ym29, ym17, ym3 + vpermb ym30, ym18, ym3 + mova m0, m8 + mova m1, m8 + mova ym2, ym8 + mova ym3, ym8 + vpdpbusd m4, m19, m10 + vpdpbusd m5, m20, m10 + vpdpbusd m6, m22, m10 + vpdpbusd m7, m23, m10 + vpdpbusd m0, m25, m10 + vpdpbusd m1, m26, m10 + vpdpbusd ym2, ym28, ym10 + vpdpbusd ym3, ym29, ym10 + vpdpbusd m4, m20, m11 + vpdpbusd m5, m21, m11 + vpdpbusd m6, m23, m11 + vpdpbusd m7, m24, m11 + vpdpbusd m0, m26, m11 + vpdpbusd m1, m27, m11 + vpdpbusd ym2, ym29, ym11 + vpdpbusd ym3, ym30, ym11 + packssdw m4, m5 + packssdw m6, m7 + packssdw m0, m1 + packssdw ym2, ym3 + psraw m4, 2 ; 0a 0b 1a 1b + psraw m6, 2 ; 2a 2b 3a 3b + psraw m0, 2 ; 4a 4b 5a 5b + psraw ym2, 2 ; 6a 6b __ __ + vshufi32x4 m5, m4, m6, q1032 ; 1a 1b 2a 2b + vshufi32x4 m7, m6, m0, q1032 ; 3a 3b 4a 4b + vshufi32x4 m1, m0, m2, q1032 ; 5a 5b 6a 6b + punpcklwd m2, m4, m5 ; 01a 01c 12a 12c + punpckhwd m3, m4, m5 ; 01b 01d 12b 12d + punpcklwd m4, m6, m7 ; 23a 23c 34a 34c + punpckhwd m5, m6, m7 ; 23b 23d 34b 34d + punpcklwd m6, m0, m1 ; 45a 45c 56a 56c + punpckhwd m7, m0, m1 ; 45b 45d 56b 56d +.hv_loop: + movu ym19, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti32x8 m19, [srcq+strideq*0], 1 + mova m20, m9 + mova m21, m9 + mova m22, m8 + mova m23, m8 + vpdpwssd m20, m2, m12 + vpdpwssd m21, m3, m12 + vpdpwssd m20, m4, m13 + vpdpwssd m21, m5, m13 + vpermb m24, m16, m19 + vpermb m25, m17, m19 + vpermb m26, m18, m19 + vpdpbusd m22, m24, m10 + vpdpbusd m23, m25, m10 + vpdpbusd m22, m25, m11 + vpdpbusd m23, m26, m11 + packssdw m22, m23 + psraw m22, 2 ; 7a 7b 8a 8b + vshufi32x4 m0, m1, m22, q1032 ; 6a 6b 7a 7b + mova m2, m4 + mova m3, m5 + mova m1, m22 + mova m4, m6 + mova m5, m7 + punpcklwd m6, m0, m1 ; 67a 67c 78a 78c + punpckhwd m7, m0, m1 ; 67b 67d 78b 78d + vpdpwssd m20, m4, m14 + vpdpwssd m21, m5, m14 + vpdpwssd m20, m6, m15 + vpdpwssd m21, m7, m15 + psrad m20, 6 + psrad m21, 6 + packssdw m20, m21 + mova [tmpq+wq*0], ym20 + vextracti32x8 [tmpq+wq*1], m20, 1 + lea tmpq, [tmpq+wq*2] + sub hd, 2 + jg .hv_loop + movzx hd, r6b + add r5, 32 + add r7, 16 + mov tmpq, r5 + mov srcq, r7 + sub r6d, 1<<8 + jg .hv_loop0 +%endif + RET +%endmacro %macro WARP_V 5 ; dst, 02, 46, 13, 57 ; Can be done using gathers, but that's terribly slow on many CPU:s lea tmp1d, [myq+deltaq*4] lea tmp2d, [myq+deltaq*1] @@ -2822,11 +4067,125 @@ phaddw m0, m10 pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13 paddd m0, m15 ; rounded 14-bit result in upper 16 bits of dword ret +%macro WRAP_YMM 1+ + INIT_YMM cpuname + %1 + INIT_ZMM cpuname +%endmacro + %macro BIDIR_FN 1 ; op +%if mmsize == 64 + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM %1 0 + vextracti32x4 xmm1, ym0, 1 + movd [dstq ], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_ret + lea dstq, [dstq+strideq*4] + pextrd [dstq ], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_ret: + RET +.w4_h16: + vpbroadcastd m7, strided + pmulld m7, [bidir_sctr_w4] + %1 0 + kxnorw k1, k1, k1 + vpscatterdd [dstq+m7]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM %1 0 + vextracti128 xmm1, ym0, 1 + movq [dstq ], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + %1_INC_PTR 2 + lea dstq, [dstq+strideq*4] +.w8_h8: + %1 0 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq ], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq ], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + %1_INC_PTR 2 + lea dstq, [dstq+strideq*4] +.w16: + %1 0 + vpermq m0, m0, q3120 + mova [dstq ], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m7, [warp_8x8_shufA] +.w32_loop: + %1 0 + %1_INC_PTR 2 + vpermq m0, m7, m0 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m7, [warp_8x8_shufA] +.w64_loop: + %1 0 + %1_INC_PTR 2 + vpermq m0, m7, m0 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m7, [warp_8x8_shufA] +.w128_loop: + %1 0 + vpermq m6, m7, m0 + %1 2 + mova [dstq+64*0], m6 + %1_INC_PTR 4 + vpermq m6, m7, m0 + mova [dstq+64*1], m6 + add dstq, strideq + dec hd + jg .w128_loop + RET +%else %1 0 lea stride3q, [strideq*3] jmp wq .w4: vextracti128 xm1, m0, 1 @@ -2847,11 +4206,11 @@ lea dstq, [dstq+strideq*4] vextracti128 xm1, m0, 1 movd [dstq ], xm0 pextrd [dstq+strideq*1], xm0, 1 movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q], xm1, 1 + pextrd [dstq+stride3q ], xm1, 1 lea dstq, [dstq+strideq*4] pextrd [dstq ], xm0, 2 pextrd [dstq+strideq*1], xm0, 3 pextrd [dstq+strideq*2], xm1, 2 pextrd [dstq+stride3q ], xm1, 3 @@ -2889,11 +4248,11 @@ %1_INC_PTR 4 %1 0 lea dstq, [dstq+strideq*2] .w32: vpermq m0, m0, q3120 - mova [dstq], m0 + mova [dstq+strideq*0], m0 %1 2 vpermq m0, m0, q3120 mova [dstq+strideq*1], m0 sub hd, 2 jg .w32_loop @@ -2928,10 +4287,11 @@ vpermq m0, m0, q3120 mova [dstq+3*32], m0 dec hd jg .w128_loop RET +%endif %endmacro %macro AVG 1 ; src_offset mova m0, [tmp1q+(%1+0)*mmsize] paddw m0, [tmp2q+(%1+0)*mmsize] @@ -2945,18 +4305,21 @@ %macro AVG_INC_PTR 1 add tmp1q, %1*mmsize add tmp2q, %1*mmsize %endmacro +%macro AVG_FN 0 cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 - lea r6, [avg_avx2_table] +%define base r6-avg %+ SUFFIX %+ _table + lea r6, [avg %+ SUFFIX %+ _table] tzcnt wd, wm movifnidn hd, hm movsxd wq, dword [r6+wq*4] - vpbroadcastd m2, [pw_1024+r6-avg_avx2_table] + vpbroadcastd m2, [base+pw_1024] add wq, r6 BIDIR_FN AVG +%endmacro %macro W_AVG 1 ; src_offset ; (a * weight + b * (16 - weight) + 128) >> 8 ; = ((a - b) * weight + (b << 4) + 128) >> 8 ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 @@ -2974,17 +4337,19 @@ packuswb m0, m1 %endmacro %define W_AVG_INC_PTR AVG_INC_PTR +%macro W_AVG_FN 0 cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 - lea r6, [w_avg_avx2_table] +%define base r6-w_avg %+ SUFFIX %+ _table + lea r6, [w_avg %+ SUFFIX %+ _table] tzcnt wd, wm movifnidn hd, hm vpbroadcastw m4, r6m ; weight movsxd wq, dword [r6+wq*4] - vpbroadcastd m5, [pw_2048+r6-w_avg_avx2_table] + vpbroadcastd m5, [base+pw_2048] psllw m4, 12 ; (weight-16) << 12 when interpreted as signed add wq, r6 cmp dword r6m, 7 jg .weight_gt7 mov r6, tmp1q @@ -2992,16 +4357,21 @@ mov tmp1q, tmp2q psubw m4, m0, m4 ; -weight mov tmp2q, r6 .weight_gt7: BIDIR_FN W_AVG +%endmacro %macro MASK 1 ; src_offset ; (a * m + b * (64 - m) + 512) >> 10 ; = ((a - b) * m + (b << 6) + 512) >> 10 ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 - vpermq m3, [maskq+(%1+0)*(mmsize/2)], q3120 +%if mmsize == 64 + vpermq m3, m8, [maskq+%1*32] +%else + vpermq m3, [maskq+%1*16], q3120 +%endif mova m0, [tmp2q+(%1+0)*mmsize] psubw m1, m0, [tmp1q+(%1+0)*mmsize] psubb m3, m4, m3 paddw m1, m1 ; (b - a) << 1 paddb m3, m3 @@ -3019,569 +4389,70 @@ packuswb m0, m1 %endmacro %macro MASK_INC_PTR 1 add maskq, %1*mmsize/2 + add tmp2q, %1*mmsize add tmp1q, %1*mmsize - add tmp2q, %1*mmsize %endmacro +%macro MASK_FN 0 cglobal mask, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 - lea r7, [mask_avx2_table] +%define base r7-mask %+ SUFFIX %+ _table + lea r7, [mask %+ SUFFIX %+ _table] tzcnt wd, wm movifnidn hd, hm mov maskq, maskmp movsxd wq, dword [r7+wq*4] pxor m4, m4 - vpbroadcastd m5, [pw_2048+r7-mask_avx2_table] +%if mmsize == 64 + mova m8, [base+bilin_v_perm64] +%endif + vpbroadcastd m5, [base+pw_2048] add wq, r7 BIDIR_FN MASK +%endmacro MASK_FN -%macro W_MASK 2-3 0 ; src_offset, mask_out, 4:4:4 - mova m0, [tmp1q+(%1+0)*mmsize] - mova m1, [tmp2q+(%1+0)*mmsize] - psubw m1, m0 +%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4 + mova m%1, [tmp1q+mmsize*%3] + mova m1, [tmp2q+mmsize*%3] + psubw m1, m%1 pabsw m%2, m1 psubusw m%2, m6, m%2 psrlw m%2, 8 ; 64 - m psllw m2, m%2, 10 pmulhw m1, m2 - paddw m0, m1 - mova m1, [tmp1q+(%1+1)*mmsize] - mova m2, [tmp2q+(%1+1)*mmsize] + paddw m%1, m1 + mova m1, [tmp1q+mmsize*%4] + mova m2, [tmp2q+mmsize*%4] psubw m2, m1 pabsw m3, m2 psubusw m3, m6, m3 +%if cpuflag(avx512icl) + vpshldw m%2, m3, 8 + psllw m3, m%2, 10 +%if %5 + psubb m%2, m5, m%2 +%endif +%else psrlw m3, 8 -%if %3 +%if %5 packuswb m%2, m3 psubb m%2, m5, m%2 vpermq m%2, m%2, q3120 %else phaddw m%2, m3 %endif psllw m3, 10 +%endif pmulhw m2, m3 paddw m1, m2 - pmulhrsw m0, m7 + pmulhrsw m%1, m7 pmulhrsw m1, m7 - packuswb m0, m1 + packuswb m%1, m1 %endmacro -cglobal w_mask_420, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_420_avx2_table - lea r7, [w_mask_420_avx2_table] - tzcnt wd, wm - movifnidn hd, hm - mov maskq, maskmp - movd xm0, r7m ; sign - movsxd wq, dword [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m7, [base+pw_2048] - movd xm8, [base+pw_258] ; 64 * 4 + 2 - pmovzxbd m9, [base+deint_shuf4] - psubw xm8, xm0 - add wq, r7 - vpbroadcastw m8, xm8 - W_MASK 0, 4 - lea stride3q, [strideq*3] - jmp wq -.w4: - vextracti128 xm1, m0, 1 - movd [dstq ], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - cmp hd, 8 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq ], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - jg .w4_h16 -.w4_end: - vextracti128 xm0, m4, 1 - vpblendd xm1, xm4, xm0, 0x05 - vpblendd xm4, xm4, xm0, 0x0a - pshufd xm1, xm1, q2301 - psubw xm4, xm8, xm4 - psubw xm4, xm1 - psrlw xm4, 2 - packuswb xm4, xm4 - movq [maskq], xm4 - RET -.w4_h16: - W_MASK 2, 5 - lea dstq, [dstq+strideq*4] - phaddd m4, m5 - vextracti128 xm1, m0, 1 - psubw m4, m8, m4 - psrlw m4, 2 - vpermd m4, m9, m4 - vextracti128 xm5, m4, 1 - packuswb xm4, xm5 - movd [dstq ], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q], xm1, 1 - lea dstq, [dstq+strideq*4] - pextrd [dstq ], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - mova [maskq], xm4 - RET -.w8_loop: - add tmp1q, 2*32 - add tmp2q, 2*32 - W_MASK 0, 4 - lea dstq, [dstq+strideq*4] - add maskq, 8 -.w8: - vextracti128 xm2, m4, 1 - vextracti128 xm1, m0, 1 - psubw xm4, xm8, xm4 - psubw xm4, xm2 - psrlw xm4, 2 - packuswb xm4, xm4 - movq [dstq ], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xm1 - movq [maskq], xm4 - sub hd, 4 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 4*32 - add tmp2q, 4*32 - W_MASK 0, 4 - lea dstq, [dstq+strideq*4] - add maskq, 16 -.w16: - vpermq m0, m0, q3120 - mova [dstq ], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - W_MASK 2, 5 - punpckhqdq m1, m4, m5 - punpcklqdq m4, m5 - psubw m1, m8, m1 - psubw m1, m4 - psrlw m1, 2 - vpermq m0, m0, q3120 - packuswb m1, m1 - vpermd m1, m9, m1 - mova [dstq+strideq*2], xm0 - vextracti128 [dstq+stride3q ], m0, 1 - mova [maskq], xm1 - sub hd, 4 - jg .w16_loop - RET -.w32_loop: - add tmp1q, 4*32 - add tmp2q, 4*32 - W_MASK 0, 4 - lea dstq, [dstq+strideq*2] - add maskq, 16 -.w32: - vpermq m0, m0, q3120 - mova [dstq], m0 - W_MASK 2, 5 - psubw m4, m8, m4 - psubw m4, m5 - psrlw m4, 2 - vpermq m0, m0, q3120 - packuswb m4, m4 - vpermd m4, m9, m4 - mova [dstq+strideq*1], m0 - mova [maskq], xm4 - sub hd, 2 - jg .w32_loop - RET -.w64_loop_even: - psubw m10, m8, m4 - psubw m11, m8, m5 - dec hd -.w64_loop: - add tmp1q, 4*32 - add tmp2q, 4*32 - W_MASK 0, 4 - add dstq, strideq -.w64: - vpermq m0, m0, q3120 - mova [dstq], m0 - W_MASK 2, 5 - vpermq m0, m0, q3120 - mova [dstq+32], m0 - test hd, 1 - jz .w64_loop_even - psubw m4, m10, m4 - psubw m5, m11, m5 - psrlw m4, 2 - psrlw m5, 2 - packuswb m4, m5 - vpermd m4, m9, m4 - mova [maskq], m4 - add maskq, 32 - dec hd - jg .w64_loop - RET -.w128_loop_even: - psubw m12, m8, m4 - psubw m13, m8, m5 - dec hd -.w128_loop: - W_MASK 0, 4 - add dstq, strideq -.w128: - vpermq m0, m0, q3120 - mova [dstq+0*32], m0 - W_MASK 2, 5 - vpermq m0, m0, q3120 - mova [dstq+1*32], m0 - add tmp1q, 8*32 - add tmp2q, 8*32 - test hd, 1 - jz .w128_even - psubw m4, m10, m4 - psubw m5, m11, m5 - psrlw m4, 2 - psrlw m5, 2 - packuswb m4, m5 - vpermd m4, m9, m4 - mova [maskq], m4 - jmp .w128_odd -.w128_even: - psubw m10, m8, m4 - psubw m11, m8, m5 -.w128_odd: - W_MASK -4, 4 - vpermq m0, m0, q3120 - mova [dstq+2*32], m0 - W_MASK -2, 5 - vpermq m0, m0, q3120 - mova [dstq+3*32], m0 - test hd, 1 - jz .w128_loop_even - psubw m4, m12, m4 - psubw m5, m13, m5 - psrlw m4, 2 - psrlw m5, 2 - packuswb m4, m5 - vpermd m4, m9, m4 - mova [maskq+32], m4 - add maskq, 64 - dec hd - jg .w128_loop - RET - -cglobal w_mask_422, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_422_avx2_table - lea r7, [w_mask_422_avx2_table] - tzcnt wd, wm - movifnidn hd, hm - mov maskq, maskmp - movd xm0, r7m ; sign - pxor m9, m9 - movsxd wq, dword [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m7, [base+pw_2048] - pmovzxbd m10, [base+deint_shuf4] - add wq, r7 - psrlw xm8, xm7, 4 ; pw_128 - psubb xm8, xm0 - vpbroadcastb m8, xm8 - W_MASK 0, 4 - lea stride3q, [strideq*3] - jmp wq -.w4: - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - cmp hd, 8 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - jg .w4_h16 -.w4_end: - vextracti128 xm5, m4, 1 - packuswb xm4, xm5 - psubb xm5, xm8, xm4 - pavgb xm5, xm9 - pshufd xm5, xm5, q3120 - mova [maskq], xm5 - RET -.w4_h16: - W_MASK 2, 5 - lea dstq, [dstq+strideq*4] - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermd m5, m10, m5 - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - mova [maskq], m5 - RET -.w8_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4 - lea dstq, [dstq+strideq*4] - add maskq, 16 -.w8: - vextracti128 xm5, m4, 1 - vextracti128 xm1, m0, 1 - packuswb xm4, xm5 - psubb xm5, xm8, xm4 - pavgb xm5, xm9 - pshufd xm5, xm5, q3120 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xm1 - mova [maskq], xm5 - sub hd, 4 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4 - lea dstq, [dstq+strideq*4] - add maskq, 32 -.w16: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - W_MASK 2, 5 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+strideq*2], xm0 - vextracti128 [dstq+stride3q ], m0, 1 - mova [maskq], m5 - sub hd, 4 - jg .w16_loop - RET -.w32_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4 - lea dstq, [dstq+strideq*2] - add maskq, 32 -.w32: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], m0 - W_MASK 2, 5 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+strideq*1], m0 - mova [maskq], m5 - sub hd, 2 - jg .w32_loop - RET -.w64_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4 - add dstq, strideq - add maskq, 32 -.w64: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - W_MASK 2, 5 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+32*1], m0 - mova [maskq], m5 - dec hd - jg .w64_loop - RET -.w128_loop: - add tmp1q, 32*8 - add tmp2q, 32*8 - W_MASK 0, 4 - add dstq, strideq - add maskq, 32*2 -.w128: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - W_MASK 2, 5 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+32*1], m0 - mova [maskq+32*0], m5 - W_MASK 4, 4 - vpermq m0, m0, q3120 - mova [dstq+32*2], m0 - W_MASK 6, 5 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+32*3], m0 - mova [maskq+32*1], m5 - dec hd - jg .w128_loop - RET - -cglobal w_mask_444, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_444_avx2_table - lea r7, [w_mask_444_avx2_table] - tzcnt wd, wm - movifnidn hd, hm - mov maskq, maskmp - movsxd wq, dword [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m7, [base+pw_2048] - vpbroadcastd m5, [base+pb_64] - add wq, r7 - W_MASK 0, 4, 1 - lea stride3q, [strideq*3] - jmp wq -.w4: - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - mova [maskq+32*0], m4 - cmp hd, 8 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - je .w4_end - W_MASK 2, 4, 1 - lea dstq, [dstq+strideq*4] - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - mova [maskq+32*1], m4 -.w4_end: - RET -.w8_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4, 1 - lea dstq, [dstq+strideq*4] - add maskq, 32 -.w8: - vextracti128 xm1, m0, 1 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xm1 - mova [maskq], m4 - sub hd, 4 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4, 1 - lea dstq, [dstq+strideq*2] - add maskq, 32 -.w16: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - mova [maskq], m4 - sub hd, 2 - jg .w16_loop - RET -.w32_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4, 1 - add dstq, strideq - add maskq, 32 -.w32: - vpermq m0, m0, q3120 - mova [dstq], m0 - mova [maskq], m4 - dec hd - jg .w32_loop - RET -.w64_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4, 1 - add dstq, strideq - add maskq, 32*2 -.w64: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - mova [maskq+32*0], m4 - W_MASK 2, 4, 1 - vpermq m0, m0, q3120 - mova [dstq+32*1], m0 - mova [maskq+32*1], m4 - dec hd - jg .w64_loop - RET -.w128_loop: - add tmp1q, 32*8 - add tmp2q, 32*8 - W_MASK 0, 4, 1 - add dstq, strideq - add maskq, 32*4 -.w128: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - mova [maskq+32*0], m4 - W_MASK 2, 4, 1 - vpermq m0, m0, q3120 - mova [dstq+32*1], m0 - mova [maskq+32*1], m4 - W_MASK 4, 4, 1 - vpermq m0, m0, q3120 - mova [dstq+32*2], m0 - mova [maskq+32*2], m4 - W_MASK 6, 4, 1 - vpermq m0, m0, q3120 - mova [dstq+32*3], m0 - mova [maskq+32*3], m4 - dec hd - jg .w128_loop - RET - cglobal blend, 3, 7, 7, dst, ds, tmp, w, h, mask %define base r6-blend_avx2_table lea r6, [blend_avx2_table] tzcnt wd, wm movifnidn hd, hm @@ -3941,57 +4812,57 @@ ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) xor r12d, r12d lea r10, [ihq-1] cmp yq, ihq - cmovl r10, yq + cmovs r10, yq test yq, yq - cmovl r10, r12 + cmovs r10, r12 imul r10, sstrideq add srcq, r10 ; ref += iclip(x, 0, iw - 1) lea r10, [iwq-1] cmp xq, iwq - cmovl r10, xq + cmovs r10, xq test xq, xq - cmovl r10, r12 + cmovs r10, r12 add srcq, r10 ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) lea bottomextq, [yq+bhq] sub bottomextq, ihq lea r3, [bhq-1] - cmovl bottomextq, r12 + cmovs bottomextq, r12 DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \ bottomext, rightext ; top_ext = iclip(-y, 0, bh - 1) neg topextq - cmovl topextq, r12 + cmovs topextq, r12 cmp bottomextq, bhq - cmovge bottomextq, r3 + cmovns bottomextq, r3 cmp topextq, bhq cmovg topextq, r3 ; right_ext = iclip(x + bw - iw, 0, bw - 1) lea rightextq, [xq+bwq] sub rightextq, iwq lea r2, [bwq-1] - cmovl rightextq, r12 + cmovs rightextq, r12 DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \ bottomext, rightext ; left_ext = iclip(-x, 0, bw - 1) neg leftextq - cmovl leftextq, r12 + cmovs leftextq, r12 cmp rightextq, bwq - cmovge rightextq, r2 + cmovns rightextq, r2 cmp leftextq, bwq - cmovge leftextq, r2 + cmovns leftextq, r2 DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \ dst, dstride, src, sstride, bottomext, rightext ; center_h = bh - top_ext - bottom_ext @@ -4010,24 +4881,20 @@ sub centerwq, r3 %macro v_loop 3 ; need_left_ext, need_right_ext, suffix .v_loop_%3: %if %1 - test leftextq, leftextq - jz .body_%3 - ; left extension xor r3, r3 vpbroadcastb m0, [srcq] .left_loop_%3: mova [dstq+r3], m0 add r3, 32 cmp r3, leftextq jl .left_loop_%3 ; body -.body_%3: lea r12, [dstq+leftextq] %endif xor r3, r3 .body_loop_%3: movu m0, [srcq+r3] @@ -4040,12 +4907,10 @@ cmp r3, centerwq jl .body_loop_%3 %if %2 ; right extension - test rightextq, rightextq - jz .body_loop_end_%3 %if %1 add r12, centerwq %else lea r12, [dstq+centerwq] %endif @@ -4055,11 +4920,10 @@ movu [r12+r3], m0 add r3, 32 cmp r3, rightextq jl .right_loop_%3 -.body_loop_end_%3: %endif add dstq, dstrideq add srcq, sstrideq dec centerhq jg .v_loop_%3 @@ -4126,6 +4990,1188 @@ jl .top_x_loop .end: RET +cextern resize_filter + +INIT_YMM avx2 +cglobal resize, 6, 14, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + vpbroadcastd m5, dxm + vpbroadcastd m8, mx0m + vpbroadcastd m6, src_wm + + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, picptr + LEA r7, $$ +%define base r7-$$ + + vpbroadcastd m3, [base+pw_m256] + vpbroadcastd m7, [base+pd_63] + vbroadcasti128 m15, [base+pb_8x0_8x8] + pmaddwd m2, m5, [base+resize_mul] ; dx*[0,1,2,3,4,5,6,7] + pslld m5, 3 ; dx*8 + pslld m6, 14 + paddd m8, m2 ; mx+[0..7]*dx + pxor m2, m2 + + ; m2 = 0, m3 = pmulhrsw constant for x=(x+64)>>7 + ; m8 = mx+[0..7]*dx, m5 = dx*8, m6 = src_w, m7 = 0x3f, m15=0,8 + +.loop_y: + xor xd, xd + mova m4, m8 ; per-line working version of mx + +.loop_x: + pmaxsd m0, m4, m2 + psrad m9, m4, 8 ; filter offset (unmasked) + pminsd m0, m6 ; iclip(mx, 0, src_w-8) + psubd m1, m4, m0 ; pshufb offset + psrad m0, 14 ; clipped src_x offset + psrad m1, 14 ; pshufb edge_emu offset + pand m9, m7 ; filter offset (masked) + + ; load source pixels - this ugly code is vpgatherdq emulation since + ; directly using vpgatherdq on Haswell is quite a bit slower :( + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vextracti128 xm0, m0, 1 + movq xm12, [srcq+r8] + movq xm13, [srcq+r10] + movhps xm12, [srcq+r9] + movhps xm13, [srcq+r11] + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vinserti128 m12, [srcq+r8], 1 + vinserti128 m13, [srcq+r10], 1 + vpbroadcastq m10, [srcq+r9] + vpbroadcastq m11, [srcq+r11] + vpblendd m12, m12, m10, 11000000b + vpblendd m13, m13, m11, 11000000b + + ; if no emulation is required, we don't need to shuffle or emulate edges + ; this also saves 2 quasi-vpgatherdqs + vptest m1, m1 + jz .filter + + movd r8d, xm1 + pextrd r9d, xm1, 1 + pextrd r10d, xm1, 2 + pextrd r11d, xm1, 3 + movsxd r8, r8d + movsxd r9, r9d + movsxd r10, r10d + movsxd r11, r11d + vextracti128 xm1, m1, 1 + movq xm14, [base+resize_shuf+4+r8] + movq xm0, [base+resize_shuf+4+r10] + movhps xm14, [base+resize_shuf+4+r9] + movhps xm0, [base+resize_shuf+4+r11] + movd r8d, xm1 + pextrd r9d, xm1, 1 + pextrd r10d, xm1, 2 + pextrd r11d, xm1, 3 + movsxd r8, r8d + movsxd r9, r9d + movsxd r10, r10d + movsxd r11, r11d + vinserti128 m14, [base+resize_shuf+4+r8], 1 + vinserti128 m0, [base+resize_shuf+4+r10], 1 + vpbroadcastq m10, [base+resize_shuf+4+r9] + vpbroadcastq m11, [base+resize_shuf+4+r11] + vpblendd m14, m14, m10, 11000000b + vpblendd m0, m0, m11, 11000000b + + paddb m14, m15 + paddb m0, m15 + pshufb m12, m14 + pshufb m13, m0 + +.filter: + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vextracti128 xm9, m9, 1 + movq xm10, [base+resize_filter+r8*8] + movq xm11, [base+resize_filter+r10*8] + movhps xm10, [base+resize_filter+r9*8] + movhps xm11, [base+resize_filter+r11*8] + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vinserti128 m10, [base+resize_filter+r8*8], 1 + vinserti128 m11, [base+resize_filter+r10*8], 1 + vpbroadcastq m14, [base+resize_filter+r9*8] + vpbroadcastq m1, [base+resize_filter+r11*8] + vpblendd m10, m10, m14, 11000000b + vpblendd m11, m11, m1, 11000000b + + pmaddubsw m12, m10 + pmaddubsw m13, m11 + phaddw m12, m13 + vextracti128 xm13, m12, 1 + phaddsw xm12, xm13 + pmulhrsw xm12, xm3 ; x=(x+64)>>7 + packuswb xm12, xm12 + movq [dstq+xq], xm12 + + paddd m4, m5 + add xd, 8 + cmp xd, dst_wd + jl .loop_x + + add dstq, dst_strideq + add srcq, src_strideq + dec hd + jg .loop_y + RET + +INIT_YMM avx2 +PREP_BILIN +PREP_8TAP +AVG_FN +W_AVG_FN +MASK_FN + +cglobal w_mask_420, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx2_table + lea r7, [w_mask_420_avx2_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + pmovzxbd m9, [base+deint_shuf4] + vpbroadcastd m8, [base+wm_420_sign+r6*4] ; 258 - sign + add wq, r7 + W_MASK 0, 4, 0, 1 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + jg .w4_h16 +.w4_end: + vextracti128 xm0, m4, 1 + vpblendd xm1, xm4, xm0, 0x05 + vpblendd xm4, xm4, xm0, 0x0a + pshufd xm1, xm1, q2301 + psubw xm4, xm8, xm4 + psubw xm4, xm1 + psrlw xm4, 2 + packuswb xm4, xm4 + movq [maskq], xm4 + RET +.w4_h16: + W_MASK 0, 5, 2, 3 + lea dstq, [dstq+strideq*4] + phaddd m4, m5 + vextracti128 xm1, m0, 1 + psubw m4, m8, m4 + psrlw m4, 2 + vpermd m4, m9, m4 + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq], xm4 + RET +.w8_loop: + add tmp1q, 2*32 + add tmp2q, 2*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 8 +.w8: + vextracti128 xm2, m4, 1 + vextracti128 xm1, m0, 1 + psubw xm4, xm8, xm4 + psubw xm4, xm2 + psrlw xm4, 2 + packuswb xm4, xm4 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + movq [maskq], xm4 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + W_MASK 0, 5, 2, 3 + punpckhqdq m1, m4, m5 + punpcklqdq m4, m5 + psubw m1, m8, m1 + psubw m1, m4 + psrlw m1, 2 + vpermq m0, m0, q3120 + packuswb m1, m1 + vpermd m1, m9, m1 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + mova [maskq], xm1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*2] + add maskq, 16 +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + W_MASK 0, 5, 2, 3 + psubw m4, m8, m4 + psubw m4, m5 + psrlw m4, 2 + vpermq m0, m0, q3120 + packuswb m4, m4 + vpermd m4, m9, m4 + mova [dstq+strideq*1], m0 + mova [maskq], xm4 + sub hd, 2 + jg .w32_loop + RET +.w64_loop_even: + psubw m10, m8, m4 + psubw m11, m8, m5 + dec hd +.w64_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + add dstq, strideq +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + test hd, 1 + jz .w64_loop_even + psubw m4, m10, m4 + psubw m5, m11, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq], m4 + add maskq, 32 + dec hd + jg .w64_loop + RET +.w128_loop_even: + psubw m12, m8, m4 + psubw m13, m8, m5 + dec hd +.w128_loop: + W_MASK 0, 4, 0, 1 + add dstq, strideq +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + add tmp1q, 8*32 + add tmp2q, 8*32 + test hd, 1 + jz .w128_even + psubw m4, m10, m4 + psubw m5, m11, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq+32*0], m4 + jmp .w128_odd +.w128_even: + psubw m10, m8, m4 + psubw m11, m8, m5 +.w128_odd: + W_MASK 0, 4, -4, -3 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + W_MASK 0, 5, -2, -1 + vpermq m0, m0, q3120 + mova [dstq+32*3], m0 + test hd, 1 + jz .w128_loop_even + psubw m4, m12, m4 + psubw m5, m13, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq+32*1], m4 + add maskq, 64 + dec hd + jg .w128_loop + RET + +cglobal w_mask_422, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx2_table + lea r7, [w_mask_422_avx2_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + pxor m9, m9 + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + pmovzxbd m10, [base+deint_shuf4] + vpbroadcastd m8, [base+wm_422_sign+r6*4] ; 128 - sign + add wq, r7 + mov maskq, maskmp + W_MASK 0, 4, 0, 1 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + jg .w4_h16 +.w4_end: + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + psubb xm5, xm8, xm4 + pavgb xm5, xm9 + pshufd xm5, xm5, q3120 + mova [maskq], xm5 + RET +.w4_h16: + W_MASK 0, 5, 2, 3 + lea dstq, [dstq+strideq*4] + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermd m5, m10, m5 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq], m5 + RET +.w8_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w8: + vextracti128 xm5, m4, 1 + vextracti128 xm1, m0, 1 + packuswb xm4, xm5 + psubb xm5, xm8, xm4 + pavgb xm5, xm9 + pshufd xm5, xm5, q3120 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + mova [maskq], xm5 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + mova [maskq], m5 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+strideq*1], m0 + mova [maskq], m5 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + add dstq, strideq + add maskq, 32 +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*1], m0 + mova [maskq], m5 + dec hd + jg .w64_loop + RET +.w128_loop: + add tmp1q, 32*8 + add tmp2q, 32*8 + W_MASK 0, 4, 0, 1 + add dstq, strideq + add maskq, 32*2 +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*1], m0 + mova [maskq+32*0], m5 + W_MASK 0, 4, 4, 5 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + W_MASK 0, 5, 6, 7 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*3], m0 + mova [maskq+32*1], m5 + dec hd + jg .w128_loop + RET + +cglobal w_mask_444, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx2_table + lea r7, [w_mask_444_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + mov maskq, maskmp + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m5, [base+pb_64] + vpbroadcastd m7, [base+pw_2048] + add wq, r7 + W_MASK 0, 4, 0, 1, 1 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + mova [maskq+32*0], m4 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + je .w4_end + W_MASK 0, 4, 2, 3, 1 + lea dstq, [dstq+strideq*4] + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq+32*1], m4 +.w4_end: + RET +.w8_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w8: + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + mova [maskq], m4 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [maskq], m4 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32 +.w32: + vpermq m0, m0, q3120 + mova [dstq], m0 + mova [maskq], m4 + dec hd + jg .w32_loop + RET +.w64_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32*2 +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + mova [maskq+32*0], m4 + W_MASK 0, 4, 2, 3, 1 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + mova [maskq+32*1], m4 + dec hd + jg .w64_loop + RET +.w128_loop: + add tmp1q, 32*8 + add tmp2q, 32*8 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32*4 +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + mova [maskq+32*0], m4 + W_MASK 0, 4, 2, 3, 1 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + mova [maskq+32*1], m4 + W_MASK 0, 4, 4, 5, 1 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + mova [maskq+32*2], m4 + W_MASK 0, 4, 6, 7, 1 + vpermq m0, m0, q3120 + mova [dstq+32*3], m0 + mova [maskq+32*3], m4 + dec hd + jg .w128_loop + RET + +%if HAVE_AVX512ICL +INIT_ZMM avx512icl +PREP_BILIN +PREP_8TAP +AVG_FN +W_AVG_FN +MASK_FN + +cglobal w_mask_420, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx512icl_table + lea r7, [w_mask_420_avx512icl_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + vpbroadcastd m9, [base+pb_m64] ; -1 << 6 + mova ym10, [base+wm_420_mask+32] + vpbroadcastd m8, [base+wm_sign_avx512+r6*8] ; (258 - sign) << 6 + add wq, r7 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + mova m5, [wm_420_perm4] + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1 + vinserti128 ym5, [wm_420_perm4+32], 1 + vpermb ym4, ym5, ym4 + vpdpbusd ym8, ym4, ym9 + vextracti128 xmm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_end: + vpermb ym8, ym10, ym8 + movq [maskq], xm8 + RET +.w4_h16: + vpbroadcastd m11, strided + pmulld m11, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + vpdpbusd m8, m4, m9 + kxnorw k1, k1, k1 + vpermb m8, m10, m8 + mova [maskq], xm8 + vpscatterdd [dstq+m11]{k1}, m0 + RET +.w8: + mova m5, [wm_420_perm8] + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1 + vinserti128 ym5, [wm_420_perm8+32], 1 + vpermb ym4, ym5, ym4 + vpdpbusd ym8, ym4, ym9 + vpermb m8, m10, m8 + mova [maskq], xm8 + vextracti128 xmm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 16 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + mova m1, m8 + vpdpbusd m1, m4, m9 + vpermb m1, m10, m1 + mova [maskq], xm1 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16: + mova m5, [wm_420_perm16] +.w16_loop: + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + mova m1, m8 + vpdpbusd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m0, q3120 + mova [maskq], xm1 + add maskq, 16 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m5, [warp_8x8_shufA] +.w32_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpbusd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + mova [maskq], xm1 + add maskq, 16 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m12, [wm_420_perm64] ; 0, 2, 4, 6, 8, 10, 12, 14 + psrlq m13, m12, 4 ; 1, 3, 5, 7, 9, 11, 13, 15 +.w64_loop: + W_MASK 0, 4, 0, 2 + W_MASK 11, 5, 1, 3 + mova m2, m8 + vpdpbusd m2, m4, m9 + mova m3, m8 + vpdpbusd m3, m5, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermt2b m2, m10, m3 + mova m1, m0 + vpermt2q m0, m12, m11 + vpermt2q m1, m13, m11 + mova [maskq], ym2 + add maskq, 32 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w64_loop + RET +.w128: + pmovzxbq m14, [wm_420_perm64] + mova m10, [wm_420_mask] + psrlq m15, m14, 4 +.w128_loop: + W_MASK 0, 12, 0, 4 + W_MASK 11, 13, 1, 5 + mova m4, m8 + vpdpbusd m4, m12, m9 + mova m5, m8 + vpdpbusd m5, m13, m9 + mova m1, m0 + vpermt2q m0, m14, m11 + vpermt2q m1, m15, m11 + mova [dstq+strideq*0+64*0], m0 + mova [dstq+strideq*1+64*0], m1 + W_MASK 0, 12, 2, 6 + W_MASK 11, 13, 3, 7 + vprold m4, 16 + vprold m5, 16 + vpdpbusd m4, m12, m9 + vpdpbusd m5, m13, m9 + add tmp1q, 512 + add tmp2q, 512 + vpermt2b m4, m10, m5 + mova m1, m0 + vpermt2q m0, m14, m11 + vpermt2q m1, m15, m11 + mova [maskq], m4 + add maskq, 64 + mova [dstq+strideq*0+64*1], m0 + mova [dstq+strideq*1+64*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w128_loop + RET + +cglobal w_mask_422, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx512icl_table + lea r7, [w_mask_422_avx512icl_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + vpbroadcastd m9, [base+pw_m128] + mova m10, [base+wm_422_mask] + vpbroadcastd m11, [base+pb_127] + add wq, r7 + vpbroadcastd m8, [base+wm_sign_avx512+4+r6*4] + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1 + movhps xm10, [wm_422_mask+16] + vpdpwssd ym8, ym4, ym9 + vpermb ym8, ym10, ym8 + vextracti128 xmm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_end: + pand xm8, xm11 + mova [maskq], xm8 + RET +.w4_h16: + vpbroadcastd m5, strided + pmulld m5, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1 + vpdpwssd m8, m4, m9 + kxnorw k1, k1, k1 + vpermb m8, m10, m8 + pand ym8, ym11 + mova [maskq], ym8 + vpscatterdd [dstq+m5]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1 + movhps xm10, [wm_422_mask+16] + vpdpwssd ym8, ym4, ym9 + vpermb ym8, ym10, ym8 + pand xm8, xm11 + mova [maskq], xm8 + vextracti128 xmm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 32 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + vpermb m1, m10, m1 + pand ym1, ym11 + mova [maskq], ym1 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 32 + lea dstq, [dstq+strideq*4] +.w16: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + vpermb m1, m10, m1 + vpermq m0, m0, q3120 + pand ym1, ym11 + mova [maskq], ym1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m5, [warp_8x8_shufA] +.w32_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + pand ym1, ym11 + mova [maskq], ym1 + add maskq, 32 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m5, [warp_8x8_shufA] +.w64_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + pand ym1, ym11 + mova [maskq], ym1 + add maskq, 32 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m13, [warp_8x8_shufA] +.w128_loop: + W_MASK 0, 4, 0, 1 + W_MASK 12, 5, 2, 3 + mova m2, m8 + vpdpwssd m2, m4, m9 + mova m3, m8 + vpdpwssd m3, m5, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermt2b m2, m10, m3 + vpermq m0, m13, m0 + vpermq m1, m13, m12 + pand m2, m11 + mova [maskq], m2 + add maskq, 64 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, strideq + dec hd + jg .w128_loop + RET + +cglobal w_mask_444, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx512icl_table + lea r7, [w_mask_444_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m5, [base+pb_64] + vpbroadcastd m7, [base+pw_2048] + mova m8, [base+wm_444_mask] + add wq, r7 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1, 1 + vinserti128 ym8, [wm_444_mask+32], 1 + vpermb ym4, ym8, ym4 + mova [maskq], ym4 + vextracti128 xmm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_end: + RET +.w4_h16: + vpbroadcastd m9, strided + pmulld m9, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + kxnorw k1, k1, k1 + mova [maskq], m4 + vpscatterdd [dstq+m9]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1, 1 + vinserti128 ym8, [wm_444_mask+32], 1 + vpermb ym4, ym8, ym4 + mova [maskq], ym4 + vextracti128 xmm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 64 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + mova [maskq], m4 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 64 + lea dstq, [dstq+strideq*4] +.w16: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + vpermq m0, m0, q3120 + mova [maskq], m4 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m9, [warp_8x8_shufA] +.w32_loop: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + add tmp1q, 128 + add tmp2q, 128 + vpermq m0, m9, m0 + mova [maskq], m4 + add maskq, 64 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m9, [warp_8x8_shufA] +.w64_loop: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + add tmp1q, 128 + add tmp2q, 128 + vpermq m0, m9, m0 + mova [maskq], m4 + add maskq, 64 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m11, [warp_8x8_shufA] +.w128_loop: + W_MASK 0, 4, 0, 1, 1 + W_MASK 10, 9, 2, 3, 1 + vpermb m4, m8, m4 + vpermb m9, m8, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermq m0, m11, m0 + vpermq m10, m11, m10 + mova [maskq+64*0], m4 + mova [maskq+64*1], m9 + add maskq, 128 + mova [dstq+64*0], m0 + mova [dstq+64*1], m10 + add dstq, strideq + dec hd + jg .w128_loop + RET + +%endif ; HAVE_AVX512ICL + %endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc_init_tmpl.c b/third_party/dav1d/src/x86/mc_init_tmpl.c --- a/third_party/dav1d/src/x86/mc_init_tmpl.c +++ b/third_party/dav1d/src/x86/mc_init_tmpl.c @@ -47,40 +47,56 @@ decl_mc_fn(dav1d_put_8tap_sharp_smooth_avx2); decl_mc_fn(dav1d_put_8tap_sharp_smooth_ssse3); decl_mc_fn(dav1d_put_bilin_avx2); decl_mc_fn(dav1d_put_bilin_ssse3); +decl_mct_fn(dav1d_prep_8tap_regular_avx512icl); decl_mct_fn(dav1d_prep_8tap_regular_avx2); decl_mct_fn(dav1d_prep_8tap_regular_ssse3); +decl_mct_fn(dav1d_prep_8tap_regular_smooth_avx512icl); decl_mct_fn(dav1d_prep_8tap_regular_smooth_avx2); decl_mct_fn(dav1d_prep_8tap_regular_smooth_ssse3); +decl_mct_fn(dav1d_prep_8tap_regular_sharp_avx512icl); decl_mct_fn(dav1d_prep_8tap_regular_sharp_avx2); decl_mct_fn(dav1d_prep_8tap_regular_sharp_ssse3); +decl_mct_fn(dav1d_prep_8tap_smooth_avx512icl); decl_mct_fn(dav1d_prep_8tap_smooth_avx2); decl_mct_fn(dav1d_prep_8tap_smooth_ssse3); +decl_mct_fn(dav1d_prep_8tap_smooth_regular_avx512icl); decl_mct_fn(dav1d_prep_8tap_smooth_regular_avx2); decl_mct_fn(dav1d_prep_8tap_smooth_regular_ssse3); +decl_mct_fn(dav1d_prep_8tap_smooth_sharp_avx512icl); decl_mct_fn(dav1d_prep_8tap_smooth_sharp_avx2); decl_mct_fn(dav1d_prep_8tap_smooth_sharp_ssse3); +decl_mct_fn(dav1d_prep_8tap_sharp_avx512icl); decl_mct_fn(dav1d_prep_8tap_sharp_avx2); decl_mct_fn(dav1d_prep_8tap_sharp_ssse3); +decl_mct_fn(dav1d_prep_8tap_sharp_regular_avx512icl); decl_mct_fn(dav1d_prep_8tap_sharp_regular_avx2); decl_mct_fn(dav1d_prep_8tap_sharp_regular_ssse3); +decl_mct_fn(dav1d_prep_8tap_sharp_smooth_avx512icl); decl_mct_fn(dav1d_prep_8tap_sharp_smooth_avx2); decl_mct_fn(dav1d_prep_8tap_sharp_smooth_ssse3); +decl_mct_fn(dav1d_prep_bilin_avx512icl); decl_mct_fn(dav1d_prep_bilin_avx2); decl_mct_fn(dav1d_prep_bilin_ssse3); +decl_avg_fn(dav1d_avg_avx512icl); decl_avg_fn(dav1d_avg_avx2); decl_avg_fn(dav1d_avg_ssse3); +decl_w_avg_fn(dav1d_w_avg_avx512icl); decl_w_avg_fn(dav1d_w_avg_avx2); decl_w_avg_fn(dav1d_w_avg_ssse3); +decl_mask_fn(dav1d_mask_avx512icl); decl_mask_fn(dav1d_mask_avx2); decl_mask_fn(dav1d_mask_ssse3); +decl_w_mask_fn(dav1d_w_mask_420_avx512icl); decl_w_mask_fn(dav1d_w_mask_420_avx2); decl_w_mask_fn(dav1d_w_mask_420_ssse3); +decl_w_mask_fn(dav1d_w_mask_422_avx512icl); decl_w_mask_fn(dav1d_w_mask_422_avx2); +decl_w_mask_fn(dav1d_w_mask_444_avx512icl); decl_w_mask_fn(dav1d_w_mask_444_avx2); decl_blend_fn(dav1d_blend_avx2); decl_blend_fn(dav1d_blend_ssse3); decl_blend_dir_fn(dav1d_blend_v_avx2); decl_blend_dir_fn(dav1d_blend_v_ssse3); @@ -97,10 +113,13 @@ decl_warp8x8t_fn(dav1d_warp_affine_8x8t_sse2); decl_emu_edge_fn(dav1d_emu_edge_avx2); decl_emu_edge_fn(dav1d_emu_edge_ssse3); +decl_resize_fn(dav1d_resize_avx2); +decl_resize_fn(dav1d_resize_ssse3); + COLD void bitfn(dav1d_mc_dsp_init_x86)(Dav1dMCDSPContext *const c) { #define init_mc_fn(type, name, suffix) \ c->mc[type] = dav1d_put_##name##_##suffix #define init_mct_fn(type, name, suffix) \ c->mct[type] = dav1d_prep_##name##_##suffix @@ -150,24 +169,26 @@ c->warp8x8 = dav1d_warp_affine_8x8_ssse3; c->warp8x8t = dav1d_warp_affine_8x8t_ssse3; c->emu_edge = dav1d_emu_edge_ssse3; + c->resize = dav1d_resize_ssse3; #endif if(!(flags & DAV1D_X86_CPU_FLAG_SSE41)) return; #if BITDEPTH == 8 c->warp8x8 = dav1d_warp_affine_8x8_sse4; c->warp8x8t = dav1d_warp_affine_8x8t_sse4; #endif +#if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; -#if BITDEPTH == 8 && ARCH_X86_64 +#if BITDEPTH == 8 init_mc_fn (FILTER_2D_8TAP_REGULAR, 8tap_regular, avx2); init_mc_fn (FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx2); init_mc_fn (FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx2); init_mc_fn (FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx2); init_mc_fn (FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx2); @@ -200,7 +221,32 @@ c->warp8x8 = dav1d_warp_affine_8x8_avx2; c->warp8x8t = dav1d_warp_affine_8x8t_avx2; c->emu_edge = dav1d_emu_edge_avx2; + c->resize = dav1d_resize_avx2; +#endif + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) + return; + +#if HAVE_AVX512ICL && BITDEPTH == 8 + init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, avx512icl); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx512icl); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx512icl); + init_mct_fn(FILTER_2D_BILINEAR, bilin, avx512icl); + + c->avg = dav1d_avg_avx512icl; + c->w_avg = dav1d_w_avg_avx512icl; + c->mask = dav1d_mask_avx512icl; + c->w_mask[0] = dav1d_w_mask_444_avx512icl; + c->w_mask[1] = dav1d_w_mask_422_avx512icl; + c->w_mask[2] = dav1d_w_mask_420_avx512icl; +#endif #endif } diff --git a/third_party/dav1d/src/x86/mc_ssse3.asm b/third_party/dav1d/src/x86/mc_ssse3.asm --- a/third_party/dav1d/src/x86/mc_ssse3.asm +++ b/third_party/dav1d/src/x86/mc_ssse3.asm @@ -55,20 +55,29 @@ subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +pb_8x0_8x8: times 8 db 0 + times 8 db 8 +resize_mul: dd 0, 1, 2, 3 +resize_shuf: times 5 db 0 + db 1, 2, 3, 4, 5, 6 + times 5+16 db 7 + pb_64: times 16 db 64 +pw_m256: times 8 dw -256 pw_8: times 8 dw 8 pw_26: times 8 dw 26 pw_34: times 8 dw 34 pw_512: times 8 dw 512 pw_1024: times 8 dw 1024 pw_2048: times 8 dw 2048 pw_6903: times 8 dw 6903 pw_8192: times 8 dw 8192 pd_32: times 4 dd 32 +pd_63: times 4 dd 63 pd_512: times 4 dd 512 pd_16384: times 4 dd 16484 pd_32768: times 4 dd 32768 pd_262144:times 4 dd 262144 @@ -1423,11 +1432,11 @@ mova m7, [base+pw_34] ; 2 + (8 << 2) add wq, base_reg jmp wq .h_w2: %if ARCH_X86_32 - and mxd, 0xff + and mxd, 0x7f %else movzx mxd, mxb %endif dec srcq mova m4, [base+subpel_h_shuf4] @@ -1453,11 +1462,11 @@ sub hd, 2 jg .h_w2_loop RET .h_w4: %if ARCH_X86_32 - and mxd, 0xff + and mxd, 0x7f %else movzx mxd, mxb %endif dec srcq movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2] @@ -1562,20 +1571,20 @@ RET .v: %if ARCH_X86_32 movzx mxd, ssb shr ssd, 16 - cmp hd, 4 - cmovle ssd, mxd + cmp hd, 6 + cmovs ssd, mxd lea ssq, [base_reg+ssq*8+subpel_filters-put_ssse3] %else %assign stack_offset org_stack_offset WIN64_SPILL_XMM 16 movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd lea myq, [base_reg+myq*8+subpel_filters-put_ssse3] %endif tzcnt r6d, wd movzx r6d, word [base_reg+r6*2+table_offset(put, _8tap_v)] mova m7, [base+pw_512] @@ -1848,18 +1857,22 @@ %undef subpel3 .hv: %assign stack_offset org_stack_offset cmp wd, 4 jg .hv_w8 - and mxd, 0xff +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif dec srcq movd m1, [base_reg+mxq*8+subpel_filters-put_ssse3+2] %if ARCH_X86_32 movzx mxd, ssb shr ssd, 16 - cmp hd, 4 - cmovle ssd, mxd + cmp hd, 6 + cmovs ssd, mxd movq m0, [base_reg+ssq*8+subpel_filters-put_ssse3] W32_RESTORE_SSQ lea r6, [ssq*3] sub srcq, r6 %define base_reg r6 @@ -1884,12 +1897,12 @@ pshufd m6, m0, q3333 mova subpelv3, m6 %else movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd movq m0, [base_reg+myq*8+subpel_filters-put_ssse3] ALLOC_STACK mmsize*14, 14 lea ss3q, [ssq*3] sub srcq, ss3q %define subpelv0 m10 @@ -2200,12 +2213,12 @@ %define accuv0 [rsp+mmsize*11] %define accuv1 [rsp+mmsize*12] movq m1, [base_reg+mxq*8+subpel_filters-put_ssse3] movzx mxd, ssb shr ssd, 16 - cmp hd, 4 - cmovle ssd, mxd + cmp hd, 6 + cmovs ssd, mxd movq m5, [base_reg+ssq*8+subpel_filters-put_ssse3] mov ssq, ssmp ALLOC_STACK -mmsize*13 %if STACK_ALIGNMENT < 16 %define srcm [rsp+mmsize*13+gprsize*1] @@ -2241,12 +2254,12 @@ %define accuv0 m8 %define accuv1 m9 movq m0, [base_reg+mxq*8+subpel_filters-put_ssse3] movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd movq m1, [base_reg+myq*8+subpel_filters-put_ssse3] pshufd subpelh0, m0, q0000 pshufd subpelh1, m0, q1111 punpcklqdq m1, m1 punpcklbw m1, m1 @@ -2509,11 +2522,11 @@ mova m7, [base+pw_8192] add wq, base_reg jmp wq .h_w4: %if ARCH_X86_32 - and mxd, 0xff + and mxd, 0x7f %else movzx mxd, mxb %endif dec srcq movd m4, [base_reg+mxq*8+subpel_filters-prep_ssse3+2] @@ -2633,19 +2646,19 @@ %endif .v: %if ARCH_X86_32 mov mxd, myd - and mxd, 0xff + and mxd, 0x7f %else %assign stack_offset org_stack_offset WIN64_SPILL_XMM 16 movzx mxd, myb %endif shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd lea myq, [base_reg+myq*8+subpel_filters-prep_ssse3] mova m2, [base+pw_512] psrlw m2, m2, 1 ; 0x0100 mova m7, [base+pw_8192] %if ARCH_X86_32 @@ -2847,18 +2860,18 @@ .hv: %assign stack_offset org_stack_offset cmp wd, 4 jg .hv_w8 - and mxd, 0xff + and mxd, 0x7f movd m1, [base_reg+mxq*8+subpel_filters-prep_ssse3+2] %if ARCH_X86_32 mov mxd, myd - and mxd, 0xff shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + and mxd, 0x7f + cmp hd, 6 + cmovs myd, mxd movq m0, [base_reg+myq*8+subpel_filters-prep_ssse3] mov r5, r2; use as new base %define base_reg r5 %assign regs_used 2 ALLOC_STACK -mmsize*14 @@ -2883,12 +2896,12 @@ pshufd m6, m0, q3333 mova subpelv3, m6 %else movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd movq m0, [base_reg+myq*8+subpel_filters-prep_ssse3] ALLOC_STACK mmsize*14, 14 lea stride3q, [strideq*3] sub srcq, stride3q dec srcq @@ -3099,15 +3112,15 @@ %define subpelv2 [rsp+mmsize*9] %define subpelv3 [rsp+mmsize*10] %define accuv0 [rsp+mmsize*11] %define accuv1 [rsp+mmsize*12] movq m1, [base_reg+mxq*8+subpel_filters-prep_ssse3] - movzx mxd, myw - and mxd, 0xff + mov mxd, myd shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + and mxd, 0x7f + cmp hd, 6 + cmovs myd, mxd movq m5, [base_reg+myq*8+subpel_filters-prep_ssse3] ALLOC_STACK -mmsize*13 %if STACK_ALIGNMENT < mmsize mov rstk, r2m %define tmpm [rsp+mmsize*13+gprsize*1] @@ -3148,12 +3161,12 @@ %define accuv0 m8 %define accuv1 m9 movq m0, [base_reg+mxq*8+subpel_filters-prep_ssse3] movzx mxd, myb shr myd, 16 - cmp hd, 4 - cmovle myd, mxd + cmp hd, 6 + cmovs myd, mxd movq m1, [base_reg+myq*8+subpel_filters-prep_ssse3] pshufd subpelh0, m0, q0000 pshufd subpelh1, m0, q1111 punpcklbw m1, m1 psraw m1, 8 ; sign-extend @@ -4741,13 +4754,13 @@ ; ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) xor reg_zero, reg_zero lea reg_tmp, [ihq-1] cmp yq, ihq - cmovl reg_tmp, yq + cmovs reg_tmp, yq test yq, yq - cmovl reg_tmp, reg_zero + cmovs reg_tmp, reg_zero %if ARCH_X86_64 imul reg_tmp, sstrideq add srcq, reg_tmp %else imul reg_tmp, sstridem @@ -4756,13 +4769,13 @@ %endif ; ; ref += iclip(x, 0, iw - 1) lea reg_tmp, [iwq-1] cmp xq, iwq - cmovl reg_tmp, xq + cmovs reg_tmp, xq test xq, xq - cmovl reg_tmp, reg_zero + cmovs reg_tmp, reg_zero add reg_src, reg_tmp %if ARCH_X86_32 mov srcm, reg_src %endif ; @@ -4771,22 +4784,22 @@ mov r1, r1m ; restore bh %endif lea reg_bottomext, [yq+bhq] sub reg_bottomext, ihq lea r3, [bhq-1] - cmovl reg_bottomext, reg_zero + cmovs reg_bottomext, reg_zero ; DEFINE_ARGS bw, bh, iw, ih, x, \ topext, dst, dstride, src, sstride, \ bottomext, rightext, blk ; top_ext = iclip(-y, 0, bh - 1) neg topextq - cmovl topextq, reg_zero + cmovs topextq, reg_zero cmp reg_bottomext, bhq - cmovge reg_bottomext, r3 + cmovns reg_bottomext, r3 cmp topextq, bhq cmovg topextq, r3 %if ARCH_X86_32 mov r4m, reg_bottomext ; @@ -4794,26 +4807,26 @@ mov r0, r0m ; restore bw %endif lea reg_rightext, [xq+bwq] sub reg_rightext, iwq lea r2, [bwq-1] - cmovl reg_rightext, reg_zero + cmovs reg_rightext, reg_zero DEFINE_ARGS bw, bh, iw, ih, leftext, \ topext, dst, dstride, src, sstride, \ bottomext, rightext, blk ; left_ext = iclip(-x, 0, bw - 1) neg leftextq - cmovl leftextq, reg_zero + cmovs leftextq, reg_zero cmp reg_rightext, bwq - cmovge reg_rightext, r2 + cmovns reg_rightext, r2 %if ARCH_X86_32 mov r3m, r1 %endif cmp leftextq, bwq - cmovge leftextq, r2 + cmovns leftextq, r2 %undef reg_zero %undef reg_tmp %undef reg_src %undef reg_bottomext @@ -4866,12 +4879,10 @@ %if ARCH_X86_32 mov r0, r0m mov r1, r1m %endif %if %1 - test leftextq, leftextq - jz .body_%3 ; left extension %if ARCH_X86_64 movd m0, [srcq] %else mov r3, srcm @@ -4883,11 +4894,10 @@ mova [dstq+r3], m0 add r3, mmsize cmp r3, leftextq jl .left_loop_%3 ; body -.body_%3: lea reg_tmp, [dstq+leftextq] %endif xor r3, r3 .body_loop_%3: %if ARCH_X86_64 @@ -4904,17 +4914,10 @@ add r3, mmsize cmp r3, centerwq jl .body_loop_%3 %if %2 ; right extension - %if ARCH_X86_64 - test rightextq, rightextq - %else - mov r1, r3m - test r1, r1 - %endif - jz .body_loop_end_%3 %if %1 add reg_tmp, centerwq %else lea reg_tmp, [dstq+centerwq] %endif @@ -4933,11 +4936,10 @@ cmp r3, rightextq %else cmp r3, r3m %endif jl .right_loop_%3 -.body_loop_end_%3: %endif %if ARCH_X86_64 add dstq, dstrideq add srcq, sstrideq dec centerhq @@ -5075,5 +5077,228 @@ RET %undef reg_dstride %undef reg_blkm %undef reg_tmp + +cextern resize_filter + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal resize, 0, 14, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%elif STACK_ALIGNMENT >= 16 +cglobal resize, 0, 7, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%else +cglobal resize, 0, 6, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%endif + movifnidn dstq, dstmp + movifnidn srcq, srcmp +%if STACK_ALIGNMENT >= 16 + movifnidn dst_wd, dst_wm +%endif +%if ARCH_X86_64 + movifnidn hd, hm +%endif + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + movd m7, dxm + movd m6, mx0m + movd m5, src_wm + pshufd m7, m7, q0000 + pshufd m6, m6, q0000 + pshufd m5, m5, q0000 + +%if ARCH_X86_64 + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, picptr + LEA r7, $$ +%define base r7-$$ +%else + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x +%if STACK_ALIGNMENT >= 16 + LEA r6, $$ +%define base r6-$$ +%else + LEA r4, $$ +%define base r4-$$ +%endif +%endif + +%if ARCH_X86_64 + mova m12, [base+pw_m256] + mova m11, [base+pd_63] + mova m10, [base+pb_8x0_8x8] +%else +%define m12 [base+pw_m256] +%define m11 [base+pd_63] +%define m10 [base+pb_8x0_8x8] +%endif + pmaddwd m4, m7, [base+resize_mul] ; dx*[0,1,2,3] + pslld m7, 2 ; dx*4 + pslld m5, 14 + paddd m6, m4 ; mx+[0..3]*dx + SCRATCH 7, 15, 0 + SCRATCH 6, 14, 1 + SCRATCH 5, 13, 2 + + ; m2 = 0, m3 = pmulhrsw constant for x=(x+64)>>7 + ; m8 = mx+[0..3]*dx, m5 = dx*4, m6 = src_w, m7 = 0x3f, m15=0,8 + +.loop_y: + xor xd, xd + mova m0, m14 ; per-line working version of mx + +.loop_x: + pxor m1, m1 + pcmpgtd m1, m0 + pandn m1, m0 + psrad m2, m0, 8 ; filter offset (unmasked) + pcmpgtd m3, m13, m1 + pand m1, m3 + pandn m3, m13 + por m1, m3 + psubd m3, m0, m1 ; pshufb offset + psrad m1, 14 ; clipped src_x offset + psrad m3, 14 ; pshufb edge_emu offset + pand m2, m11 ; filter offset (masked) + + ; load source pixels +%if ARCH_X86_64 + movd r8d, xm1 + pshuflw xm1, xm1, q3232 + movd r9d, xm1 + punpckhqdq xm1, xm1 + movd r10d, xm1 + psrlq xm1, 32 + movd r11d, xm1 + movq xm4, [srcq+r8] + movq xm5, [srcq+r10] + movhps xm4, [srcq+r9] + movhps xm5, [srcq+r11] +%else + movd r3d, xm1 + pshufd xm1, xm1, q3312 + movd r1d, xm1 + pshuflw xm1, xm1, q3232 + movq xm4, [srcq+r3] + movq xm5, [srcq+r1] + movd r3d, xm1 + punpckhqdq xm1, xm1 + movd r1d, xm1 + movhps xm4, [srcq+r3] + movhps xm5, [srcq+r1] +%endif + + ; if no emulation is required, we don't need to shuffle or emulate edges + ; this also saves 2 quasi-vpgatherdqs + pxor m6, m6 + pcmpeqb m6, m3 +%if ARCH_X86_64 + pmovmskb r8d, m6 + cmp r8d, 0xffff +%else + pmovmskb r3d, m6 + cmp r3d, 0xffff +%endif + je .filter + +%if ARCH_X86_64 + movd r8d, xm3 + pshuflw xm3, xm3, q3232 + movd r9d, xm3 + punpckhqdq xm3, xm3 + movd r10d, xm3 + psrlq xm3, 32 + movd r11d, xm3 + movsxd r8, r8d + movsxd r9, r9d + movsxd r10, r10d + movsxd r11, r11d + movq xm6, [base+resize_shuf+4+r8] + movq xm7, [base+resize_shuf+4+r10] + movhps xm6, [base+resize_shuf+4+r9] + movhps xm7, [base+resize_shuf+4+r11] +%else + movd r3d, xm3 + pshufd xm3, xm3, q3312 + movd r1d, xm3 + pshuflw xm3, xm3, q3232 + movq xm6, [base+resize_shuf+4+r3] + movq xm7, [base+resize_shuf+4+r1] + movd r3d, xm3 + punpckhqdq xm3, xm3 + movd r1d, xm3 + movhps xm6, [base+resize_shuf+4+r3] + movhps xm7, [base+resize_shuf+4+r1] +%endif + + paddb m6, m10 + paddb m7, m10 + pshufb m4, m6 + pshufb m5, m7 + +.filter: +%if ARCH_X86_64 + movd r8d, xm2 + pshuflw xm2, xm2, q3232 + movd r9d, xm2 + punpckhqdq xm2, xm2 + movd r10d, xm2 + psrlq xm2, 32 + movd r11d, xm2 + movq xm6, [base+resize_filter+r8*8] + movq xm7, [base+resize_filter+r10*8] + movhps xm6, [base+resize_filter+r9*8] + movhps xm7, [base+resize_filter+r11*8] +%else + movd r3d, xm2 + pshufd xm2, xm2, q3312 + movd r1d, xm2 + pshuflw xm2, xm2, q3232 + movq xm6, [base+resize_filter+r3*8] + movq xm7, [base+resize_filter+r1*8] + movd r3d, xm2 + punpckhqdq xm2, xm2 + movd r1d, xm2 + movhps xm6, [base+resize_filter+r3*8] + movhps xm7, [base+resize_filter+r1*8] +%endif + + pmaddubsw m4, m6 + pmaddubsw m5, m7 + phaddw m4, m5 + phaddsw m4, m4 + pmulhrsw m4, m12 ; x=(x+64)>>7 + packuswb m4, m4 + movd [dstq+xq], m4 + + paddd m0, m15 + add xd, 4 +%if STACK_ALIGNMENT >= 16 + cmp xd, dst_wd +%else + cmp xd, dst_wm +%endif + jl .loop_x + +%if ARCH_X86_64 + add dstq, dst_strideq + add srcq, src_strideq + dec hd +%else + add dstq, dst_stridem + add srcq, src_stridem + dec dword r5m +%endif + jg .loop_y + RET diff --git a/third_party/dav1d/src/x86/msac.h b/third_party/dav1d/src/x86/msac.h --- a/third_party/dav1d/src/x86/msac.h +++ b/third_party/dav1d/src/x86/msac.h @@ -37,17 +37,28 @@ unsigned dav1d_msac_decode_bool_adapt_sse2(MsacContext *s, uint16_t *cdf); unsigned dav1d_msac_decode_bool_equi_sse2(MsacContext *s); unsigned dav1d_msac_decode_bool_sse2(MsacContext *s, unsigned f); unsigned dav1d_msac_decode_hi_tok_sse2(MsacContext *s, uint16_t *cdf); +/* Needed for checkasm */ +unsigned dav1d_msac_decode_symbol_adapt16_avx2(MsacContext *s, uint16_t *cdf, + size_t n_symbols); + #if ARCH_X86_64 || defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) #define dav1d_msac_decode_symbol_adapt4 dav1d_msac_decode_symbol_adapt4_sse2 #define dav1d_msac_decode_symbol_adapt8 dav1d_msac_decode_symbol_adapt8_sse2 -#define dav1d_msac_decode_symbol_adapt16 dav1d_msac_decode_symbol_adapt16_sse2 #define dav1d_msac_decode_hi_tok dav1d_msac_decode_hi_tok_sse2 #endif #define dav1d_msac_decode_bool_adapt dav1d_msac_decode_bool_adapt_sse2 #define dav1d_msac_decode_bool_equi dav1d_msac_decode_bool_equi_sse2 #define dav1d_msac_decode_bool dav1d_msac_decode_bool_sse2 +#if ARCH_X86_64 +#define dav1d_msac_decode_symbol_adapt16(ctx, cdf, symb) ((ctx)->symbol_adapt16(ctx, cdf, symb)) +#elif defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +#define dav1d_msac_decode_symbol_adapt16 dav1d_msac_decode_symbol_adapt16_sse2 +#endif + +void dav1d_msac_init_x86(MsacContext *const s); + #endif /* DAV1D_SRC_X86_MSAC_H */ diff --git a/third_party/dav1d/src/x86/msac.asm b/third_party/dav1d/src/x86/msac.asm --- a/third_party/dav1d/src/x86/msac.asm +++ b/third_party/dav1d/src/x86/msac.asm @@ -65,11 +65,11 @@ .rng: resd 1 .cnt: resd 1 .update_cdf: resd 1 endstruc -%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) +%define m(x, y) mangle(private_prefix %+ _ %+ x %+ y) SECTION .text %if WIN64 DECLARE_REG_TMP 0, 1, 2, 3, 4, 5, 7, 3, 8 @@ -165,11 +165,11 @@ %if ARCH_X86_64 == 0 push t5 %endif lea t5, [t2+gprsize] cmp t5, rcx - jg .refill_eob + ja .refill_eob mov t2, [t2] lea ecx, [t1+23] add t1d, 16 shr ecx, 3 ; shift_bytes bswap t2 @@ -193,11 +193,11 @@ mov t5, rcx mov ecx, gprsize*8-24 sub ecx, t1d ; c .refill_eob_loop: cmp t2, t5 - jge .refill_eob_end ; eob reached + jae .refill_eob_end ; eob reached movzx t1d, byte [t2] inc t2 shl t1, cl xor t4, t1 sub ecx, 8 @@ -238,11 +238,11 @@ psubusw m1, m3 pxor m2, m2 pcmpeqw m1, m2 pmovmskb eax, m1 test t3d, t3d - jz m(msac_decode_symbol_adapt4).renorm + jz m(msac_decode_symbol_adapt4, SUFFIX).renorm movzx t3d, word [t1+t4*2] pcmpeqw m2, m2 mov t2d, t3d shr t3d, 4 cmp t4d, 3 ; may be called with n_symbols <= 2 @@ -255,11 +255,11 @@ psubw m0, m1 psraw m2, m3 paddw m0, m2 mova [t1], m0 mov [t1+t4*2], t2w - jmp m(msac_decode_symbol_adapt4).renorm + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm cglobal msac_decode_symbol_adapt16, 0, 6, 6 DECODE_SYMBOL_ADAPT_INIT LEA rax, pw_0xff00 movd m4, [t0+msac.rng] @@ -328,11 +328,11 @@ movzx t1d, word [buf+rax*2] movzx t2d, word [buf+rax*2-2] %if WIN64 add rsp, 48 %endif - jmp m(msac_decode_symbol_adapt4).renorm2 + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm2 cglobal msac_decode_bool_adapt, 0, 6, 0 movifnidn t1, r1mp movifnidn t0, r0mp movzx eax, word [t1] @@ -364,11 +364,11 @@ %if ARCH_X86_64 == 0 movzx eax, al %endif not t4 test t3d, t3d - jz m(msac_decode_symbol_adapt4).renorm3 + jz m(msac_decode_symbol_adapt4, SUFFIX).renorm3 %if UNIX64 == 0 push t6 %endif movzx t6d, word [t1+2] %if ARCH_X86_64 == 0 @@ -388,17 +388,17 @@ sub t5d, t6d ; cdf[0] -= cdf[0] >> rate; mov [t1], t5w %if WIN64 mov t1d, [t7+msac.cnt] pop t6 - jmp m(msac_decode_symbol_adapt4).renorm4 + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm4 %else %if ARCH_X86_64 == 0 pop t5 pop t6 %endif - jmp m(msac_decode_symbol_adapt4).renorm3 + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm3 %endif cglobal msac_decode_bool_equi, 0, 6, 0 movifnidn t0, r0mp mov t1d, [t0+msac.rng] @@ -416,11 +416,11 @@ setb al ; the upper 32 bits contains garbage but that's OK not t4 %if ARCH_X86_64 == 0 movzx eax, al %endif - jmp m(msac_decode_symbol_adapt4).renorm3 + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm3 cglobal msac_decode_bool, 0, 6, 0 movifnidn t0, r0mp movifnidn t1d, r1m movzx eax, byte [t0+msac.rng+1] ; r >> 8 @@ -440,11 +440,11 @@ setb al not t4 %if ARCH_X86_64 == 0 movzx eax, al %endif - jmp m(msac_decode_symbol_adapt4).renorm3 + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm3 %macro HI_TOK 1 ; update_cdf %if ARCH_X86_64 == 0 mov eax, -24 %endif @@ -596,5 +596,73 @@ test eax, eax jz .no_update_cdf HI_TOK 1 .no_update_cdf: HI_TOK 0 + +%if ARCH_X86_64 +INIT_YMM avx2 +cglobal msac_decode_symbol_adapt16, 3, 6, 6 + lea rax, [pw_0xff00] + vpbroadcastw m2, [t0+msac.rng] + mova m0, [t1] + vpbroadcastw m3, [t0+msac.dif+6] + vbroadcasti128 m4, [rax] + mov t3d, [t0+msac.update_cdf] + mov t4d, t2d + not t2 +%if STACK_ALIGNMENT < 32 + mov r5, rsp +%if WIN64 + and rsp, ~31 + sub rsp, 40 +%else + and r5, ~31 + %define buf r5-32 +%endif +%elif WIN64 + sub rsp, 64 +%else + %define buf rsp-56 +%endif + psrlw m1, m0, 6 + movd [buf-4], xm2 + pand m2, m4 + psllw m1, 7 + pmulhuw m1, m2 + paddw m1, [rax+t2*2] + mova [buf], m1 + pmaxuw m1, m3 + pcmpeqw m1, m3 + pmovmskb eax, m1 + test t3d, t3d + jz .renorm + movzx t3d, word [t1+t4*2] + pcmpeqw m2, m2 + lea t2d, [t3+80] + shr t2d, 4 + cmp t3d, 32 + adc t3d, 0 + movd xm3, t2d + pavgw m2, m1 + psubw m2, m0 + psubw m0, m1 + psraw m2, xm3 + paddw m0, m2 + mova [t1], m0 + mov [t1+t4*2], t3w +.renorm: + tzcnt eax, eax + mov t4, [t0+msac.dif] + movzx t1d, word [buf+rax-0] + movzx t2d, word [buf+rax-2] + shr eax, 1 +%if WIN64 +%if STACK_ALIGNMENT < 32 + mov rsp, r5 +%else + add rsp, 64 +%endif +%endif + vzeroupper + jmp m(msac_decode_symbol_adapt4, _sse2).renorm2 +%endif diff --git a/third_party/dav1d/src/x86/msac_init.c b/third_party/dav1d/src/x86/msac_init.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/msac_init.c @@ -0,0 +1,42 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/msac.h" +#include "src/x86/msac.h" + +void dav1d_msac_init_x86(MsacContext *const s) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (flags & DAV1D_X86_CPU_FLAG_SSE2) { + s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_sse2; + } + + if (flags & DAV1D_X86_CPU_FLAG_AVX2) { + s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_avx2; + } +} + diff --git a/third_party/dav1d/tests/checkasm/cdef.c b/third_party/dav1d/tests/checkasm/cdef.c --- a/third_party/dav1d/tests/checkasm/cdef.c +++ b/third_party/dav1d/tests/checkasm/cdef.c @@ -33,76 +33,86 @@ #include "common/dump.h" #include "src/levels.h" #include "src/cdef.h" -static void init_tmp(pixel *buf, int n, const int bitdepth_max) { - while (n--) - *buf++ = rnd() & bitdepth_max; +static int to_binary(int x) { /* 0-15 -> 0000-1111 */ + return (x & 1) + 5 * (x & 2) + 25 * (x & 4) + 125 * (x & 8); } -static void check_cdef_filter(const cdef_fn fn, const int w, const int h, - const char *const name) -{ - ALIGN_STK_32(pixel, src, 10 * 16 + 8, ); - ALIGN_STK_32(pixel, c_src, 10 * 16 + 8, ), *const c_src_ptr = c_src + 8; - ALIGN_STK_32(pixel, a_src, 10 * 16 + 8, ), *const a_src_ptr = a_src + 8; - ALIGN_STK_32(pixel, top, 16 * 2 + 8, ), *const top_ptr = top + 8; - pixel left[8][2]; +static void init_tmp(pixel *buf, int n, const int bitdepth_max) { + const int fill_type = rnd() & 7; + if (fill_type == 0) + while (n--) /* check for cdef_filter underflows */ + *buf++ = rnd() & 1; + else if (fill_type == 1) + while (n--) /* check for cdef_filter overflows */ + *buf++ = bitdepth_max - (rnd() & 1); + else + while (n--) + *buf++ = rnd() & bitdepth_max; +} + +static void check_cdef_filter(const cdef_fn fn, const int w, const int h) { + ALIGN_STK_64(pixel, c_src, 16 * 10 + 16, ), *const c_dst = c_src + 8; + ALIGN_STK_64(pixel, a_src, 16 * 10 + 16, ), *const a_dst = a_src + 8; + ALIGN_STK_64(pixel, top_buf, 16 * 2 + 16, ), *const top = top_buf + 8; + ALIGN_STK_16(pixel, left, 8,[2]); + const ptrdiff_t stride = 16 * sizeof(pixel); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel (*left)[2], - pixel *const top[2], int pri_strength, int sec_strength, + const pixel *top, int pri_strength, int sec_strength, int dir, int damping, enum CdefEdgeFlags edges HIGHBD_DECL_SUFFIX); - if (check_func(fn, "%s_%dbpc", name, BITDEPTH)) { + if (check_func(fn, "cdef_filter_%dx%d_%dbpc", w, h, BITDEPTH)) { for (int dir = 0; dir < 8; dir++) { - for (enum CdefEdgeFlags edges = 0; edges <= 0xf; edges++) { + for (enum CdefEdgeFlags edges = 0x0; edges <= 0xf; edges++) { #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #else const int bitdepth_max = 0xff; #endif const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; - init_tmp(src, 10 * 16 + 8, bitdepth_max); - init_tmp(top, 16 * 2 + 8, bitdepth_max); - init_tmp((pixel *) left,8 * 2, bitdepth_max); - memcpy(a_src, src, (10 * 16 + 8) * sizeof(pixel)); - memcpy(c_src, src, (10 * 16 + 8) * sizeof(pixel)); + init_tmp(c_src, 16 * 10 + 16, bitdepth_max); + init_tmp(top_buf, 16 * 2 + 16, bitdepth_max); + init_tmp((pixel *) left, 8 * 2, bitdepth_max); + memcpy(a_src, c_src, (16 * 10 + 16) * sizeof(pixel)); const int lvl = 1 + (rnd() % 62); const int damping = 3 + (rnd() & 3) + bitdepth_min_8 - (w == 4 || (rnd() & 1)); - const int pri_strength = (lvl >> 2) << bitdepth_min_8; + int pri_strength = (lvl >> 2) << bitdepth_min_8; int sec_strength = lvl & 3; sec_strength += sec_strength == 3; sec_strength <<= bitdepth_min_8; - call_ref(c_src_ptr, 16 * sizeof(pixel), left, - (pixel *[2]) { top_ptr, top_ptr + 16 }, - pri_strength, sec_strength, dir, damping, edges - HIGHBD_TAIL_SUFFIX); - call_new(a_src_ptr, 16 * sizeof(pixel), left, - (pixel *[2]) { top_ptr, top_ptr + 16 }, - pri_strength, sec_strength, dir, damping, edges - HIGHBD_TAIL_SUFFIX); - checkasm_check_pixel(c_src, 16 * sizeof(pixel), - a_src, 16 * sizeof(pixel), - 16, 10, "src"); - checkasm_check_pixel(c_src + 16 * 10, 16 * sizeof(pixel), - a_src + 16 * 10, 16 * sizeof(pixel), - 8, 1, "src last row"); - bench_new(a_src_ptr, 16 * sizeof(pixel), left, - (pixel *[2]) { top_ptr, top_ptr + 16 }, - pri_strength, sec_strength, dir, damping, edges - HIGHBD_TAIL_SUFFIX); + call_ref(c_dst, stride, left, top, pri_strength, sec_strength, + dir, damping, edges HIGHBD_TAIL_SUFFIX); + call_new(a_dst, stride, left, top, pri_strength, sec_strength, + dir, damping, edges HIGHBD_TAIL_SUFFIX); + if (checkasm_check_pixel(c_dst, stride, a_dst, stride, w, h, "dst")) { + fprintf(stderr, "strength = %d:%d, dir = %d, damping = %d, edges = %04d\n", + pri_strength, sec_strength, dir, damping, to_binary(edges)); + return; + } + if (dir == 7 && (edges == 0x5 || edges == 0xa || edges == 0xf)) { + /* Benchmark a fixed set of cases to get consistent results: + * 1) top/left edges and pri_strength only + * 2) bottom/right edges and sec_strength only + * 3) all edges and both pri_strength and sec_strength + */ + pri_strength = (edges & 1) << bitdepth_min_8; + sec_strength = (edges & 2) << bitdepth_min_8; + bench_new(a_dst, stride, left, top, pri_strength, sec_strength, + dir, damping, edges HIGHBD_TAIL_SUFFIX); + } } } } - report(name); } static void check_cdef_direction(const cdef_dir_fn fn) { - ALIGN_STK_32(pixel, src, 8 * 8,); + ALIGN_STK_64(pixel, src, 8 * 8,); declare_func(int, pixel *src, ptrdiff_t dst_stride, unsigned *var HIGHBD_DECL_SUFFIX); if (check_func(fn, "cdef_dir_%dbpc", BITDEPTH)) { @@ -127,13 +137,14 @@ report("cdef_dir"); } void bitfn(checkasm_check_cdef)(void) { Dav1dCdefDSPContext c; - bitfn(dav1d_cdef_dsp_init)(&c); check_cdef_direction(c.dir); - check_cdef_filter(c.fb[0], 8, 8, "cdef_filter_8x8"); - check_cdef_filter(c.fb[1], 4, 8, "cdef_filter_4x8"); - check_cdef_filter(c.fb[2], 4, 4, "cdef_filter_4x4"); + + check_cdef_filter(c.fb[0], 8, 8); + check_cdef_filter(c.fb[1], 4, 8); + check_cdef_filter(c.fb[2], 4, 4); + report("cdef_filter"); } diff --git a/third_party/dav1d/tests/checkasm/checkasm.h b/third_party/dav1d/tests/checkasm/checkasm.h --- a/third_party/dav1d/tests/checkasm/checkasm.h +++ b/third_party/dav1d/tests/checkasm/checkasm.h @@ -191,16 +191,24 @@ * misbehaving functions to either produce incorrect output or segfault. Note * that even though this works extremely well in practice, it's technically * not guaranteed and false negatives is theoretically possible, but there * can never be any false positives. */ void checkasm_stack_clobber(uint64_t clobber, ...); +/* YMM and ZMM registers on x86 are turned off to save power when they haven't + * been used for some period of time. When they are used there will be a + * "warmup" period during which performance will be reduced and inconsistent + * which is problematic when trying to benchmark individual functions. We can + * work around this by periodically issuing "dummy" instructions that uses + * those registers to keep them powered on. */ +void checkasm_simd_warmup(void); #define declare_new(ret, ...)\ ret (*checked_call)(void *, int, int, int, int, int, __VA_ARGS__) =\ (void *)checkasm_checked_call; #define CLOB (UINT64_C(0xdeadbeefdeadbeef)) #define call_new(...)\ (checkasm_set_signal_handler_state(1),\ + checkasm_simd_warmup(),\ checkasm_stack_clobber(CLOB, CLOB, CLOB, CLOB, CLOB, CLOB, CLOB,\ CLOB, CLOB, CLOB, CLOB, CLOB, CLOB, CLOB,\ CLOB, CLOB, CLOB, CLOB, CLOB, CLOB, CLOB),\ checked_call(func_new, 0, 0, 0, 0, 0, __VA_ARGS__));\ checkasm_set_signal_handler_state(0) diff --git a/third_party/dav1d/tests/checkasm/checkasm.c b/third_party/dav1d/tests/checkasm/checkasm.c --- a/third_party/dav1d/tests/checkasm/checkasm.c +++ b/third_party/dav1d/tests/checkasm/checkasm.c @@ -96,23 +96,19 @@ const char *name; const char *suffix; unsigned flag; } cpus[] = { #if ARCH_X86 - { "SSE", "sse", DAV1D_X86_CPU_FLAG_SSE }, - { "SSE2", "sse2", DAV1D_X86_CPU_FLAG_SSE2 }, - { "SSE3", "sse3", DAV1D_X86_CPU_FLAG_SSE3 }, - { "SSSE3", "ssse3", DAV1D_X86_CPU_FLAG_SSSE3 }, - { "SSE4.1", "sse4", DAV1D_X86_CPU_FLAG_SSE41 }, - { "SSE4.2", "sse42", DAV1D_X86_CPU_FLAG_SSE42 }, - { "AVX", "avx", DAV1D_X86_CPU_FLAG_AVX }, - { "AVX2", "avx2", DAV1D_X86_CPU_FLAG_AVX2 }, - { "AVX-512", "avx512", DAV1D_X86_CPU_FLAG_AVX512 }, + { "SSE2", "sse2", DAV1D_X86_CPU_FLAG_SSE2 }, + { "SSSE3", "ssse3", DAV1D_X86_CPU_FLAG_SSSE3 }, + { "SSE4.1", "sse4", DAV1D_X86_CPU_FLAG_SSE41 }, + { "AVX2", "avx2", DAV1D_X86_CPU_FLAG_AVX2 }, + { "AVX-512 (Ice Lake)", "avx512icl", DAV1D_X86_CPU_FLAG_AVX512ICL }, #elif ARCH_AARCH64 || ARCH_ARM - { "NEON", "neon", DAV1D_ARM_CPU_FLAG_NEON }, + { "NEON", "neon", DAV1D_ARM_CPU_FLAG_NEON }, #elif ARCH_PPC64LE - { "VSX", "vsx", DAV1D_PPC_CPU_FLAG_VSX }, + { "VSX", "vsx", DAV1D_PPC_CPU_FLAG_VSX }, #endif { 0 } }; typedef struct CheckasmFuncVersion { @@ -148,10 +144,13 @@ const char *test_name; unsigned int seed; int bench_c; int verbose; int function_listing; +#if ARCH_X86_64 + void (*simd_warmup)(void); +#endif } state; /* float compare support code */ typedef union { float f; @@ -567,17 +566,30 @@ argv++; } fprintf(stderr, "checkasm: using random seed %u\n", state.seed); + dav1d_init_cpu(); +#if ARCH_X86_64 + void checkasm_warmup_avx2(void); + void checkasm_warmup_avx512(void); + unsigned cpu_flags = dav1d_get_cpu_flags(); + if (cpu_flags & DAV1D_X86_CPU_FLAG_AVX512ICL) + state.simd_warmup = checkasm_warmup_avx512; + else if (cpu_flags & DAV1D_X86_CPU_FLAG_AVX2) + state.simd_warmup = checkasm_warmup_avx2; + else + state.simd_warmup = NULL; + checkasm_simd_warmup(); +#endif check_cpu_flag(NULL, 0); + if (state.function_listing) { print_functions(state.funcs); } else { for (int i = 0; cpus[i].flag; i++) check_cpu_flag(cpus[i].name, cpus[i].flag); - if (!state.num_checked) { fprintf(stderr, "checkasm: no tests to perform\n"); } else if (state.num_failed) { fprintf(stderr, "checkasm: %d of %d tests have failed\n", state.num_failed, state.num_checked); @@ -772,5 +784,13 @@ DEF_CHECKASM_CHECK_FUNC(uint8_t, "%02x") DEF_CHECKASM_CHECK_FUNC(uint16_t, "%04x") DEF_CHECKASM_CHECK_FUNC(int16_t, "%6d") DEF_CHECKASM_CHECK_FUNC(int32_t, "%9d") + +#if ARCH_X86_64 +void checkasm_simd_warmup(void) +{ + if (state.simd_warmup) + state.simd_warmup(); +} +#endif diff --git a/third_party/dav1d/tests/checkasm/filmgrain.c b/third_party/dav1d/tests/checkasm/filmgrain.c --- a/third_party/dav1d/tests/checkasm/filmgrain.c +++ b/third_party/dav1d/tests/checkasm/filmgrain.c @@ -47,33 +47,33 @@ declare_func(void, entry grain_lut[][GRAIN_WIDTH], const Dav1dFilmGrainData *data HIGHBD_DECL_SUFFIX); for (int i = 0; i < 4; i++) { if (check_func(dsp->generate_grain_y, "gen_grain_y_ar%d_%dbpc", i, BITDEPTH)) { - Dav1dFilmGrainData fg_data; - fg_data.seed = rnd() & 0xFFFF; + ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,); + fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #endif - fg_data.grain_scale_shift = rnd() & 3; - fg_data.ar_coeff_shift = (rnd() & 3) + 6; - fg_data.ar_coeff_lag = i; - const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1); + fg_data[0].grain_scale_shift = rnd() & 3; + fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; + fg_data[0].ar_coeff_lag = i; + const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++) - fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128; + fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128; - call_ref(grain_lut_c, &fg_data HIGHBD_TAIL_SUFFIX); - call_new(grain_lut_a, &fg_data HIGHBD_TAIL_SUFFIX); + call_ref(grain_lut_c, fg_data HIGHBD_TAIL_SUFFIX); + call_new(grain_lut_a, fg_data HIGHBD_TAIL_SUFFIX); if (memcmp(grain_lut_c, grain_lut_a, GRAIN_WIDTH * GRAIN_HEIGHT * sizeof(entry))) { fail(); } - bench_new(grain_lut_a, &fg_data HIGHBD_TAIL_SUFFIX); + bench_new(grain_lut_a, fg_data HIGHBD_TAIL_SUFFIX); } } report("gen_grain_y"); } @@ -95,124 +95,124 @@ for (int i = 0; i < 4; i++) { if (check_func(dsp->generate_grain_uv[layout_idx], "gen_grain_uv_ar%d_%dbpc_%s", i, BITDEPTH, ss_name[layout_idx])) { - Dav1dFilmGrainData fg_data; - fg_data.seed = rnd() & 0xFFFF; + ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,); + fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #endif - fg_data.num_y_points = rnd() & 1; - fg_data.grain_scale_shift = rnd() & 3; - fg_data.ar_coeff_shift = (rnd() & 3) + 6; - fg_data.ar_coeff_lag = i; - const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1); + fg_data[0].num_y_points = rnd() & 1; + fg_data[0].grain_scale_shift = rnd() & 3; + fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; + fg_data[0].ar_coeff_lag = i; + const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++) - fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128; - dsp->generate_grain_y(grain_lut_y, &fg_data HIGHBD_TAIL_SUFFIX); + fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128; + dsp->generate_grain_y(grain_lut_y, fg_data HIGHBD_TAIL_SUFFIX); const int uv = rnd() & 1; - const int num_uv_pos = num_y_pos + !!fg_data.num_y_points; + const int num_uv_pos = num_y_pos + !!fg_data[0].num_y_points; for (int n = 0; n < num_uv_pos; n++) - fg_data.ar_coeffs_uv[uv][n] = (rnd() & 0xff) - 128; - if (!fg_data.num_y_points) - fg_data.ar_coeffs_uv[uv][num_uv_pos] = 0; + fg_data[0].ar_coeffs_uv[uv][n] = (rnd() & 0xff) - 128; + if (!fg_data[0].num_y_points) + fg_data[0].ar_coeffs_uv[uv][num_uv_pos] = 0; memset(grain_lut_c, 0xff, sizeof(grain_lut_c)); memset(grain_lut_a, 0xff, sizeof(grain_lut_a)); - call_ref(grain_lut_c, grain_lut_y, &fg_data, uv HIGHBD_TAIL_SUFFIX); - call_new(grain_lut_a, grain_lut_y, &fg_data, uv HIGHBD_TAIL_SUFFIX); + call_ref(grain_lut_c, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX); + call_new(grain_lut_a, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX); int diff = 0, w = ss_x ? 44 : GRAIN_WIDTH; for (int y = 0; y < (ss_y ? 38 : GRAIN_HEIGHT); y++) diff |= memcmp(grain_lut_a[y], grain_lut_c[y], w * sizeof(entry)); if (diff) fail(); - bench_new(grain_lut_a, grain_lut_y, &fg_data, uv HIGHBD_TAIL_SUFFIX); + bench_new(grain_lut_a, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX); } } } report("gen_grain_uv"); } static void check_fgy_sbrow(const Dav1dFilmGrainDSPContext *const dsp) { - ALIGN_STK_32(pixel, c_dst, 128 * 32,); - ALIGN_STK_32(pixel, a_dst, 128 * 32,); - ALIGN_STK_32(pixel, src, 128 * 32,); + ALIGN_STK_64(pixel, c_dst, 128 * 32,); + ALIGN_STK_64(pixel, a_dst, 128 * 32,); + ALIGN_STK_64(pixel, src, 128 * 32,); const ptrdiff_t stride = 128 * sizeof(pixel); declare_func(void, pixel *dst_row, const pixel *src_row, ptrdiff_t stride, const Dav1dFilmGrainData *data, size_t pw, const uint8_t scaling[SCALING_SIZE], const entry grain_lut[][GRAIN_WIDTH], int bh, int row_num HIGHBD_DECL_SUFFIX); if (check_func(dsp->fgy_32x32xn, "fgy_32x32xn_%dbpc", BITDEPTH)) { - Dav1dFilmGrainData fg_data; - fg_data.seed = rnd() & 0xFFFF; + ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 16,); + fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #else const int bitdepth_max = 0xff; #endif uint8_t scaling[SCALING_SIZE]; entry grain_lut[GRAIN_HEIGHT + 1][GRAIN_WIDTH]; - fg_data.grain_scale_shift = rnd() & 3; - fg_data.ar_coeff_shift = (rnd() & 3) + 6; - fg_data.ar_coeff_lag = rnd() & 3; - const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1); + fg_data[0].grain_scale_shift = rnd() & 3; + fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; + fg_data[0].ar_coeff_lag = rnd() & 3; + const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++) - fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128; - dsp->generate_grain_y(grain_lut, &fg_data HIGHBD_TAIL_SUFFIX); + fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128; + dsp->generate_grain_y(grain_lut, fg_data HIGHBD_TAIL_SUFFIX); - fg_data.num_y_points = 2 + (rnd() % 13); - const int pad = 0xff / fg_data.num_y_points; - for (int n = 0; n < fg_data.num_y_points; n++) { - fg_data.y_points[n][0] = 0xff * n / fg_data.num_y_points; - fg_data.y_points[n][0] += rnd() % pad; - fg_data.y_points[n][1] = rnd() & 0xff; + fg_data[0].num_y_points = 2 + (rnd() % 13); + const int pad = 0xff / fg_data[0].num_y_points; + for (int n = 0; n < fg_data[0].num_y_points; n++) { + fg_data[0].y_points[n][0] = 0xff * n / fg_data[0].num_y_points; + fg_data[0].y_points[n][0] += rnd() % pad; + fg_data[0].y_points[n][1] = rnd() & 0xff; } - generate_scaling(bitdepth_from_max(bitdepth_max), fg_data.y_points, - fg_data.num_y_points, scaling); + generate_scaling(bitdepth_from_max(bitdepth_max), fg_data[0].y_points, + fg_data[0].num_y_points, scaling); const int w = 1 + (rnd() & 127); const int h = 1 + (rnd() & 31); for (int y = 0; y < h; y++) for (int x = 0; x < w; x++) src[y * PXSTRIDE(stride) + x] = rnd() & bitdepth_max; const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0; - fg_data.clip_to_restricted_range = rnd() & 1; - fg_data.scaling_shift = (rnd() & 3) + 8; - for (fg_data.overlap_flag = 0; fg_data.overlap_flag <= 1; - fg_data.overlap_flag++) + fg_data[0].clip_to_restricted_range = rnd() & 1; + fg_data[0].scaling_shift = (rnd() & 3) + 8; + for (fg_data[0].overlap_flag = 0; fg_data[0].overlap_flag <= 1; + fg_data[0].overlap_flag++) { - call_ref(c_dst, src, stride, &fg_data, w, scaling, grain_lut, h, + call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut, h, row_num HIGHBD_TAIL_SUFFIX); - call_new(a_dst, src, stride, &fg_data, w, scaling, grain_lut, h, + call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut, h, row_num HIGHBD_TAIL_SUFFIX); checkasm_check_pixel(c_dst, stride, a_dst, stride, w, h, "dst"); } - fg_data.overlap_flag = 1; - bench_new(a_dst, src, stride, &fg_data, 64, scaling, grain_lut, 32, + fg_data[0].overlap_flag = 1; + bench_new(a_dst, src, stride, fg_data, 64, scaling, grain_lut, 32, row_num HIGHBD_TAIL_SUFFIX); } report("fgy_32x32xn"); } static void check_fguv_sbrow(const Dav1dFilmGrainDSPContext *const dsp) { - ALIGN_STK_32(pixel, c_dst, 128 * 32,); - ALIGN_STK_32(pixel, a_dst, 128 * 32,); - ALIGN_STK_32(pixel, src, 128 * 32,); - ALIGN_STK_32(pixel, luma_src, 128 * 32,); + ALIGN_STK_64(pixel, c_dst, 128 * 32,); + ALIGN_STK_64(pixel, a_dst, 128 * 32,); + ALIGN_STK_64(pixel, src, 128 * 32,); + ALIGN_STK_64(pixel, luma_src, 128 * 32,); const ptrdiff_t lstride = 128 * sizeof(pixel); declare_func(void, pixel *dst_row, const pixel *src_row, ptrdiff_t stride, const Dav1dFilmGrainData *data, size_t pw, const uint8_t scaling[SCALING_SIZE], @@ -229,13 +229,13 @@ for (int csfl = 0; csfl <= 1; csfl++) { if (check_func(dsp->fguv_32x32xn[layout_idx], "fguv_32x32xn_%dbpc_%s_csfl%d", BITDEPTH, ss_name[layout_idx], csfl)) { - Dav1dFilmGrainData fg_data; + ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,); - fg_data.seed = rnd() & 0xFFFF; + fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #else const int bitdepth_max = 0xff; @@ -243,19 +243,22 @@ const int uv_pl = rnd() & 1; const int is_identity = rnd() & 1; uint8_t scaling[SCALING_SIZE]; entry grain_lut[2][GRAIN_HEIGHT + 1][GRAIN_WIDTH]; - fg_data.grain_scale_shift = rnd() & 3; - fg_data.ar_coeff_shift = (rnd() & 3) + 6; - fg_data.ar_coeff_lag = rnd() & 3; - const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1); + fg_data[0].grain_scale_shift = rnd() & 3; + fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; + fg_data[0].ar_coeff_lag = rnd() & 3; + const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++) - fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128; - dsp->generate_grain_y(grain_lut[0], &fg_data HIGHBD_TAIL_SUFFIX); + fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128; + const int num_uv_pos = num_y_pos + 1; + for (int n = 0; n < num_uv_pos; n++) + fg_data[0].ar_coeffs_uv[uv_pl][n] = (rnd() & 0xff) - 128; + dsp->generate_grain_y(grain_lut[0], fg_data HIGHBD_TAIL_SUFFIX); dsp->generate_grain_uv[layout_idx](grain_lut[1], grain_lut[0], - &fg_data, uv_pl HIGHBD_TAIL_SUFFIX); + fg_data, uv_pl HIGHBD_TAIL_SUFFIX); const int w = 1 + (rnd() & (127 >> ss_x)); const int h = 1 + (rnd() & (31 >> ss_y)); const int lw = w << ss_x, lh = h << ss_y; @@ -266,51 +269,51 @@ for (int x = 0; x < lw; x++) luma_src[y * PXSTRIDE(lstride) + x] = rnd() & bitdepth_max; const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0; if (csfl) { - fg_data.num_y_points = 2 + (rnd() % 13); - const int pad = 0xff / fg_data.num_y_points; - for (int n = 0; n < fg_data.num_y_points; n++) { - fg_data.y_points[n][0] = 0xff * n / fg_data.num_y_points; - fg_data.y_points[n][0] += rnd() % pad; - fg_data.y_points[n][1] = rnd() & 0xff; + fg_data[0].num_y_points = 2 + (rnd() % 13); + const int pad = 0xff / fg_data[0].num_y_points; + for (int n = 0; n < fg_data[0].num_y_points; n++) { + fg_data[0].y_points[n][0] = 0xff * n / fg_data[0].num_y_points; + fg_data[0].y_points[n][0] += rnd() % pad; + fg_data[0].y_points[n][1] = rnd() & 0xff; } - generate_scaling(bitdepth_from_max(bitdepth_max), fg_data.y_points, - fg_data.num_y_points, scaling); + generate_scaling(bitdepth_from_max(bitdepth_max), fg_data[0].y_points, + fg_data[0].num_y_points, scaling); } else { - fg_data.num_uv_points[uv_pl] = 2 + (rnd() % 9); - const int pad = 0xff / fg_data.num_uv_points[uv_pl]; - for (int n = 0; n < fg_data.num_uv_points[uv_pl]; n++) { - fg_data.uv_points[uv_pl][n][0] = 0xff * n / fg_data.num_uv_points[uv_pl]; - fg_data.uv_points[uv_pl][n][0] += rnd() % pad; - fg_data.uv_points[uv_pl][n][1] = rnd() & 0xff; + fg_data[0].num_uv_points[uv_pl] = 2 + (rnd() % 9); + const int pad = 0xff / fg_data[0].num_uv_points[uv_pl]; + for (int n = 0; n < fg_data[0].num_uv_points[uv_pl]; n++) { + fg_data[0].uv_points[uv_pl][n][0] = 0xff * n / fg_data[0].num_uv_points[uv_pl]; + fg_data[0].uv_points[uv_pl][n][0] += rnd() % pad; + fg_data[0].uv_points[uv_pl][n][1] = rnd() & 0xff; } - generate_scaling(bitdepth_from_max(bitdepth_max), fg_data.uv_points[uv_pl], - fg_data.num_uv_points[uv_pl], scaling); + generate_scaling(bitdepth_from_max(bitdepth_max), fg_data[0].uv_points[uv_pl], + fg_data[0].num_uv_points[uv_pl], scaling); - fg_data.uv_mult[uv_pl] = (rnd() & 0xff) - 128; - fg_data.uv_luma_mult[uv_pl] = (rnd() & 0xff) - 128; - fg_data.uv_offset[uv_pl] = (rnd() & 0x1ff) - 256; + fg_data[0].uv_mult[uv_pl] = (rnd() & 0xff) - 128; + fg_data[0].uv_luma_mult[uv_pl] = (rnd() & 0xff) - 128; + fg_data[0].uv_offset[uv_pl] = (rnd() & 0x1ff) - 256; } - fg_data.clip_to_restricted_range = rnd() & 1; - fg_data.scaling_shift = (rnd() & 3) + 8; - fg_data.chroma_scaling_from_luma = csfl; - for (fg_data.overlap_flag = 0; fg_data.overlap_flag <= 1; - fg_data.overlap_flag++) + fg_data[0].clip_to_restricted_range = rnd() & 1; + fg_data[0].scaling_shift = (rnd() & 3) + 8; + fg_data[0].chroma_scaling_from_luma = csfl; + for (fg_data[0].overlap_flag = 0; fg_data[0].overlap_flag <= 1; + fg_data[0].overlap_flag++) { - call_ref(c_dst, src, stride, &fg_data, w, scaling, grain_lut[1], h, + call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut[1], h, row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); - call_new(a_dst, src, stride, &fg_data, w, scaling, grain_lut[1], h, + call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut[1], h, row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); checkasm_check_pixel(c_dst, stride, a_dst, stride, w, h, "dst"); } - fg_data.overlap_flag = 1; - bench_new(a_dst, src, stride, &fg_data, 32, scaling, grain_lut[1], 16, + fg_data[0].overlap_flag = 1; + bench_new(a_dst, src, stride, fg_data, 32, scaling, grain_lut[1], 16, row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); } } } diff --git a/third_party/dav1d/tests/checkasm/ipred.c b/third_party/dav1d/tests/checkasm/ipred.c --- a/third_party/dav1d/tests/checkasm/ipred.c +++ b/third_party/dav1d/tests/checkasm/ipred.c @@ -64,94 +64,108 @@ 58, 61, 64, 67, 70, 73, 76, 81, 84, 87 }; static void check_intra_pred(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 64 * 64,); - ALIGN_STK_32(pixel, a_dst, 64 * 64,); - ALIGN_STK_32(pixel, topleft_buf, 257,); + ALIGN_STK_64(pixel, c_dst, 64 * 64,); + ALIGN_STK_64(pixel, a_dst, 64 * 64,); + ALIGN_STK_64(pixel, topleft_buf, 257,); pixel *const topleft = topleft_buf + 128; declare_func(void, pixel *dst, ptrdiff_t stride, const pixel *topleft, int width, int height, int angle, int max_width, int max_height HIGHBD_DECL_SUFFIX); - for (int mode = 0; mode < N_IMPL_INTRA_PRED_MODES; mode++) - for (int w = 4; w <= (mode == FILTER_PRED ? 32 : 64); w <<= 1) - if (check_func(c->intra_pred[mode], "intra_pred_%s_w%d_%dbpc", - intra_pred_mode_names[mode], w, BITDEPTH)) - { - for (int h = imax(w / 4, 4); h <= imin(w * 4, - (mode == FILTER_PRED ? 32 : 64)); h <<= 1) + for (int mode = 0; mode < N_IMPL_INTRA_PRED_MODES; mode++) { + int bpc_min = BITDEPTH, bpc_max = BITDEPTH; + if (mode == FILTER_PRED && BITDEPTH == 16) { + bpc_min = 10; + bpc_max = 12; + } + for (int bpc = bpc_min; bpc <= bpc_max; bpc += 2) + for (int w = 4; w <= (mode == FILTER_PRED ? 32 : 64); w <<= 1) + if (check_func(c->intra_pred[mode], "intra_pred_%s_w%d_%dbpc", + intra_pred_mode_names[mode], w, bpc)) { - const ptrdiff_t stride = w * sizeof(pixel); + for (int h = imax(w / 4, 4); h <= imin(w * 4, + (mode == FILTER_PRED ? 32 : 64)); h <<= 1) + { + const ptrdiff_t stride = w * sizeof(pixel); - int a = 0, maxw = 0, maxh = 0; - if (mode >= Z1_PRED && mode <= Z3_PRED) { /* angle */ - a = (90 * (mode - Z1_PRED) + z_angles[rnd() % 27]) | - (rnd() & 0x600); - if (mode == Z2_PRED) { - maxw = rnd(), maxh = rnd(); - maxw = 1 + (maxw & (maxw & 4096 ? 4095 : w - 1)); - maxh = 1 + (maxh & (maxh & 4096 ? 4095 : h - 1)); - } - } else if (mode == FILTER_PRED) /* filter_idx */ - a = (rnd() % 5) | (rnd() & ~511); + int a = 0, maxw = 0, maxh = 0; + if (mode >= Z1_PRED && mode <= Z3_PRED) { /* angle */ + a = (90 * (mode - Z1_PRED) + z_angles[rnd() % 27]) | + (rnd() & 0x600); + if (mode == Z2_PRED) { + maxw = rnd(), maxh = rnd(); + maxw = 1 + (maxw & (maxw & 4096 ? 4095 : w - 1)); + maxh = 1 + (maxh & (maxh & 4096 ? 4095 : h - 1)); + } + } else if (mode == FILTER_PRED) /* filter_idx */ + a = (rnd() % 5) | (rnd() & ~511); -#if BITDEPTH == 16 - const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; -#else - const int bitdepth_max = 0xff; -#endif + int bitdepth_max; + if (bpc == 16) + bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; + else + bitdepth_max = (1 << bpc) - 1; - for (int i = -h * 2; i <= w * 2; i++) - topleft[i] = rnd() & bitdepth_max; + for (int i = -h * 2; i <= w * 2; i++) + topleft[i] = rnd() & bitdepth_max; - call_ref(c_dst, stride, topleft, w, h, a, maxw, maxh - HIGHBD_TAIL_SUFFIX); - call_new(a_dst, stride, topleft, w, h, a, maxw, maxh - HIGHBD_TAIL_SUFFIX); - if (checkasm_check_pixel(c_dst, stride, a_dst, stride, - w, h, "dst")) - { - if (mode == Z1_PRED || mode == Z3_PRED) - fprintf(stderr, "angle = %d (0x%03x)\n", - a & 0x1ff, a & 0x600); - else if (mode == Z2_PRED) - fprintf(stderr, "angle = %d (0x%03x), " - "max_width = %d, max_height = %d\n", - a & 0x1ff, a & 0x600, maxw, maxh); - else if (mode == FILTER_PRED) - fprintf(stderr, "filter_idx = %d\n", a & 0x1ff); + call_ref(c_dst, stride, topleft, w, h, a, maxw, maxh + HIGHBD_TAIL_SUFFIX); + call_new(a_dst, stride, topleft, w, h, a, maxw, maxh + HIGHBD_TAIL_SUFFIX); + if (checkasm_check_pixel(c_dst, stride, a_dst, stride, + w, h, "dst")) + { + if (mode == Z1_PRED || mode == Z3_PRED) + fprintf(stderr, "angle = %d (0x%03x)\n", + a & 0x1ff, a & 0x600); + else if (mode == Z2_PRED) + fprintf(stderr, "angle = %d (0x%03x), " + "max_width = %d, max_height = %d\n", + a & 0x1ff, a & 0x600, maxw, maxh); + else if (mode == FILTER_PRED) + fprintf(stderr, "filter_idx = %d\n", a & 0x1ff); + } + + bench_new(a_dst, stride, topleft, w, h, a, 128, 128 + HIGHBD_TAIL_SUFFIX); } - - bench_new(a_dst, stride, topleft, w, h, a, 128, 128 - HIGHBD_TAIL_SUFFIX); } - } + } report("intra_pred"); } static void check_cfl_ac(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(int16_t, c_dst, 32 * 32,); - ALIGN_STK_32(int16_t, a_dst, 32 * 32,); - ALIGN_STK_32(pixel, luma, 32 * 32,); + ALIGN_STK_64(int16_t, c_dst, 32 * 32,); + ALIGN_STK_64(int16_t, a_dst, 32 * 32,); + ALIGN_STK_64(pixel, luma, 32 * 32,); declare_func(void, int16_t *ac, const pixel *y, ptrdiff_t stride, int w_pad, int h_pad, int cw, int ch); for (int layout = 1; layout <= DAV1D_PIXEL_LAYOUT_I444; layout++) { const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_step = 2 >> ss_hor, v_step = 2 >> ss_ver; for (int w = 4; w <= (32 >> ss_hor); w <<= 1) if (check_func(c->cfl_ac[layout - 1], "cfl_ac_%s_w%d_%dbpc", cfl_ac_names[layout - 1], w, BITDEPTH)) { - for (int h = imax(w / 4, 4); h <= imin(w * 4, (32 >> ss_ver)); h <<= 1) { + for (int h = imax(w / 4, 4); + h <= imin(w * 4, (32 >> ss_ver)); h <<= 1) + { const ptrdiff_t stride = 32 * sizeof(pixel); - for (int w_pad = (w >> 2) - 1; w_pad >= 0; w_pad--) { - for (int h_pad = (h >> 2) - 1; h_pad >= 0; h_pad--) { + for (int w_pad = imax((w >> 2) - h_step, 0); + w_pad >= 0; w_pad -= h_step) + { + for (int h_pad = imax((h >> 2) - v_step, 0); + h_pad >= 0; h_pad -= v_step) + { #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #else const int bitdepth_max = 0xff; #endif @@ -173,14 +187,14 @@ } report("cfl_ac"); } static void check_cfl_pred(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 32 * 32,); - ALIGN_STK_32(pixel, a_dst, 32 * 32,); - ALIGN_STK_32(int16_t, ac, 32 * 32,); - ALIGN_STK_32(pixel, topleft_buf, 257,); + ALIGN_STK_64(pixel, c_dst, 32 * 32,); + ALIGN_STK_64(pixel, a_dst, 32 * 32,); + ALIGN_STK_64(int16_t, ac, 32 * 32,); + ALIGN_STK_64(pixel, topleft_buf, 257,); pixel *const topleft = topleft_buf + 128; declare_func(void, pixel *dst, ptrdiff_t stride, const pixel *topleft, int width, int height, const int16_t *ac, int alpha HIGHBD_DECL_SUFFIX); @@ -225,13 +239,13 @@ } report("cfl_pred"); } static void check_pal_pred(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 64 * 64,); - ALIGN_STK_32(pixel, a_dst, 64 * 64,); - ALIGN_STK_32(uint8_t, idx, 64 * 64,); + ALIGN_STK_64(pixel, c_dst, 64 * 64,); + ALIGN_STK_64(pixel, a_dst, 64 * 64,); + ALIGN_STK_64(uint8_t, idx, 64 * 64,); ALIGN_STK_16(uint16_t, pal, 8,); declare_func(void, pixel *dst, ptrdiff_t stride, const uint16_t *pal, const uint8_t *idx, int w, int h); diff --git a/third_party/dav1d/tests/checkasm/itx.c b/third_party/dav1d/tests/checkasm/itx.c --- a/third_party/dav1d/tests/checkasm/itx.c +++ b/third_party/dav1d/tests/checkasm/itx.c @@ -224,13 +224,13 @@ void bitfn(checkasm_check_itx)(void) { Dav1dInvTxfmDSPContext c; bitfn(dav1d_itx_dsp_init)(&c); - ALIGN_STK_32(coef, coeff, 2, [32 * 32]); - ALIGN_STK_32(pixel, c_dst, 64 * 64,); - ALIGN_STK_32(pixel, a_dst, 64 * 64,); + ALIGN_STK_64(coef, coeff, 2, [32 * 32]); + ALIGN_STK_64(pixel, c_dst, 64 * 64,); + ALIGN_STK_64(pixel, a_dst, 64 * 64,); static const uint8_t txfm_size_order[N_RECT_TX_SIZES] = { TX_4X4, RTX_4X8, RTX_4X16, RTX_8X4, TX_8X8, RTX_8X16, RTX_8X32, RTX_16X4, RTX_16X8, TX_16X16, RTX_16X32, RTX_16X64, diff --git a/third_party/dav1d/tests/checkasm/loopfilter.c b/third_party/dav1d/tests/checkasm/loopfilter.c --- a/third_party/dav1d/tests/checkasm/loopfilter.c +++ b/third_party/dav1d/tests/checkasm/loopfilter.c @@ -93,12 +93,12 @@ static void check_lpf_sb(loopfilter_sb_fn fn, const char *const name, const int n_blks, const int lf_idx, const int is_chroma, const int dir) { - ALIGN_STK_32(pixel, c_dst_mem, 128 * 16,); - ALIGN_STK_32(pixel, a_dst_mem, 128 * 16,); + ALIGN_STK_64(pixel, c_dst_mem, 128 * 16,); + ALIGN_STK_64(pixel, a_dst_mem, 128 * 16,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const uint32_t *mask, const uint8_t (*l)[4], ptrdiff_t b4_stride, const Av1FilterLUT *lut, int w HIGHBD_DECL_SUFFIX); diff --git a/third_party/dav1d/tests/checkasm/looprestoration.c b/third_party/dav1d/tests/checkasm/looprestoration.c --- a/third_party/dav1d/tests/checkasm/looprestoration.c +++ b/third_party/dav1d/tests/checkasm/looprestoration.c @@ -41,14 +41,14 @@ buf[x] = rnd() & bitdepth_max; buf += PXSTRIDE(stride); } } -static void check_wiener(Dav1dLoopRestorationDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 448 * 64,); - ALIGN_STK_32(pixel, a_dst, 448 * 64,); - ALIGN_STK_32(pixel, h_edge, 448 * 8,); +static void check_wiener(Dav1dLoopRestorationDSPContext *const c, const int bpc) { + ALIGN_STK_64(pixel, c_dst, 448 * 64,); + ALIGN_STK_64(pixel, a_dst, 448 * 64,); + ALIGN_STK_64(pixel, h_edge, 448 * 8,); pixel left[64][4]; declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel (*const left)[4], const pixel *lpf, ptrdiff_t lpf_stride, @@ -56,11 +56,11 @@ const int16_t filterv[7], enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX); for (int pl = 0; pl < 2; pl++) { if (check_func(c->wiener, "wiener_%s_%dbpc", - pl ? "chroma" : "luma", BITDEPTH)) + pl ? "chroma" : "luma", bpc)) { int16_t filter[2][3], filter_v[7], filter_h[7]; filter[0][0] = pl ? 0 : (rnd() & 15) - 5; filter[0][1] = (rnd() & 31) - 23; @@ -79,15 +79,11 @@ filter_v[2] = filter_v[4] = filter[1][2]; filter_v[3] = -((filter_v[0] + filter_v[1] + filter_v[2]) * 2); const int base_w = 1 + (rnd() % 384); const int base_h = 1 + (rnd() & 63); -#if BITDEPTH == 16 - const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; -#else - const int bitdepth_max = 0xff; -#endif + const int bitdepth_max = (1 << bpc) - 1; init_tmp(c_dst, 448 * sizeof(pixel), 448, 64, bitdepth_max); init_tmp(h_edge, 448 * sizeof(pixel), 448, 8, bitdepth_max); init_tmp((pixel *) left, 4 * sizeof(pixel), 4, 64, bitdepth_max); @@ -110,17 +106,16 @@ bench_new(a_dst + 32, 448 * sizeof(pixel), left, h_edge + 32, 448 * sizeof(pixel), 256, 64, filter_h, filter_v, 0xf HIGHBD_TAIL_SUFFIX); } } - report("wiener"); } -static void check_sgr(Dav1dLoopRestorationDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 448 * 64,); - ALIGN_STK_32(pixel, a_dst, 448 * 64,); - ALIGN_STK_32(pixel, h_edge, 448 * 8,); +static void check_sgr(Dav1dLoopRestorationDSPContext *const c, const int bpc) { + ALIGN_STK_64(pixel, c_dst, 448 * 64,); + ALIGN_STK_64(pixel, a_dst, 448 * 64,); + ALIGN_STK_64(pixel, h_edge, 448 * 8,); pixel left[64][4]; declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel (*const left)[4], const pixel *lpf, ptrdiff_t lpf_stride, @@ -128,25 +123,21 @@ const int16_t sgr_wt[7], enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX); for (int sgr_idx = 14; sgr_idx >= 6; sgr_idx -= 4) { if (check_func(c->selfguided, "selfguided_%s_%dbpc", - sgr_idx == 6 ? "mix" : sgr_idx == 10 ? "3x3" : "5x5", BITDEPTH)) + sgr_idx == 6 ? "mix" : sgr_idx == 10 ? "3x3" : "5x5", bpc)) { int16_t sgr_wt[2]; sgr_wt[0] = dav1d_sgr_params[sgr_idx][0] ? (rnd() & 127) - 96 : 0; sgr_wt[1] = dav1d_sgr_params[sgr_idx][1] ? (rnd() & 127) - 32 : iclip(128 - sgr_wt[0], -32, 95); const int base_w = 1 + (rnd() % 384); const int base_h = 1 + (rnd() & 63); -#if BITDEPTH == 16 - const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; -#else - const int bitdepth_max = 0xff; -#endif + const int bitdepth_max = (1 << bpc) - 1; init_tmp(c_dst, 448 * sizeof(pixel), 448, 64, bitdepth_max); init_tmp(h_edge, 448 * sizeof(pixel), 448, 8, bitdepth_max); init_tmp((pixel *) left, 4 * sizeof(pixel), 4, 64, bitdepth_max); @@ -169,16 +160,26 @@ bench_new(a_dst + 32, 448 * sizeof(pixel), left, h_edge + 32, 448 * sizeof(pixel), 256, 64, sgr_idx, sgr_wt, 0xf HIGHBD_TAIL_SUFFIX); } } - report("sgr"); } void bitfn(checkasm_check_looprestoration)(void) { - Dav1dLoopRestorationDSPContext c; - - bitfn(dav1d_loop_restoration_dsp_init)(&c); - - check_wiener(&c); - check_sgr(&c); +#if BITDEPTH == 16 + const int bpc_min = 10, bpc_max = 12; +#else + const int bpc_min = 8, bpc_max = 8; +#endif + for (int bpc = bpc_min; bpc <= bpc_max; bpc += 2) { + Dav1dLoopRestorationDSPContext c; + bitfn(dav1d_loop_restoration_dsp_init)(&c, bpc); + check_wiener(&c, bpc); + } + report("wiener"); + for (int bpc = bpc_min; bpc <= bpc_max; bpc += 2) { + Dav1dLoopRestorationDSPContext c; + bitfn(dav1d_loop_restoration_dsp_init)(&c, bpc); + check_sgr(&c, bpc); + } + report("sgr"); } diff --git a/third_party/dav1d/tests/checkasm/mc.c b/third_party/dav1d/tests/checkasm/mc.c --- a/third_party/dav1d/tests/checkasm/mc.c +++ b/third_party/dav1d/tests/checkasm/mc.c @@ -53,13 +53,13 @@ return h * 2; } } static void check_mc(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 135 * 135,); - ALIGN_STK_32(pixel, c_dst, 128 * 128,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(pixel, src_buf, 135 * 135,); + ALIGN_STK_64(pixel, c_dst, 128 * 128,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); const pixel *src = src_buf + 135 * 3 + 3; const ptrdiff_t src_stride = 135 * sizeof(pixel); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *src, ptrdiff_t src_stride, int w, int h, int mx, int my @@ -116,13 +116,13 @@ buf[135*y+x] = ((x | y) < 8 ? (pattern[x] ^ pattern[y] ^ sign) : rnd()) & bitdepth_max; } static void check_mct(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 135 * 135,); - ALIGN_STK_32(int16_t, c_tmp, 128 * 128,); - ALIGN_STK_32(int16_t, a_tmp, 128 * 128,); + ALIGN_STK_64(pixel, src_buf, 135 * 135,); + ALIGN_STK_64(int16_t, c_tmp, 128 * 128,); + ALIGN_STK_64(int16_t, a_tmp, 128 * 128,); const pixel *src = src_buf + 135 * 3 + 3; const ptrdiff_t src_stride = 135 * sizeof(pixel); declare_func(void, int16_t *tmp, const pixel *src, ptrdiff_t src_stride, int w, int h, int mx, int my HIGHBD_DECL_SUFFIX); @@ -171,13 +171,13 @@ 8, 8 HIGHBD_TAIL_SUFFIX); } } static void check_avg(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const int16_t *tmp1, const int16_t *tmp2, int w, int h HIGHBD_DECL_SUFFIX); for (int w = 4; w <= 128; w <<= 1) @@ -202,13 +202,13 @@ } report("avg"); } static void check_w_avg(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const int16_t *tmp1, const int16_t *tmp2, int w, int h, int weight HIGHBD_DECL_SUFFIX); for (int w = 4; w <= 128; w <<= 1) @@ -234,14 +234,14 @@ } report("w_avg"); } static void check_mask(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); - ALIGN_STK_32(uint8_t, mask, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(uint8_t, mask, 128 * 128,); for (int i = 0; i < 128 * 128; i++) mask[i] = rnd() % 65; declare_func(void, pixel *dst, ptrdiff_t dst_stride, const int16_t *tmp1, @@ -269,15 +269,15 @@ } report("mask"); } static void check_w_mask(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(int16_t, tmp, 2, [128 * 128]); - ALIGN_STK_32(pixel, c_dst, 135 * 135,); - ALIGN_STK_32(pixel, a_dst, 128 * 128,); - ALIGN_STK_32(uint8_t, c_mask, 128 * 128,); - ALIGN_STK_32(uint8_t, a_mask, 128 * 128,); + ALIGN_STK_64(int16_t, tmp, 2, [128 * 128]); + ALIGN_STK_64(pixel, c_dst, 135 * 135,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); + ALIGN_STK_64(uint8_t, c_mask, 128 * 128,); + ALIGN_STK_64(uint8_t, a_mask, 128 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const int16_t *tmp1, const int16_t *tmp2, int w, int h, uint8_t *mask, int sign HIGHBD_DECL_SUFFIX); @@ -319,14 +319,14 @@ } report("w_mask"); } static void check_blend(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, tmp, 32 * 32,); - ALIGN_STK_32(pixel, c_dst, 32 * 32,); - ALIGN_STK_32(pixel, a_dst, 32 * 32,); - ALIGN_STK_32(uint8_t, mask, 32 * 32,); + ALIGN_STK_64(pixel, tmp, 32 * 32,); + ALIGN_STK_64(pixel, c_dst, 32 * 32,); + ALIGN_STK_64(pixel, a_dst, 32 * 32,); + ALIGN_STK_64(uint8_t, mask, 32 * 32,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, int w, int h, const uint8_t *mask); for (int w = 4; w <= 32; w <<= 1) { @@ -355,13 +355,13 @@ } report("blend"); } static void check_blend_v(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, tmp, 32 * 128,); - ALIGN_STK_32(pixel, c_dst, 32 * 128,); - ALIGN_STK_32(pixel, a_dst, 32 * 128,); + ALIGN_STK_64(pixel, tmp, 32 * 128,); + ALIGN_STK_64(pixel, c_dst, 32 * 128,); + ALIGN_STK_64(pixel, a_dst, 32 * 128,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, int w, int h); for (int w = 2; w <= 32; w <<= 1) { @@ -389,13 +389,13 @@ } report("blend_v"); } static void check_blend_h(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, tmp, 128 * 32,); - ALIGN_STK_32(pixel, c_dst, 128 * 32,); - ALIGN_STK_32(pixel, a_dst, 128 * 32,); + ALIGN_STK_64(pixel, tmp, 128 * 32,); + ALIGN_STK_64(pixel, c_dst, 128 * 32,); + ALIGN_STK_64(pixel, a_dst, 128 * 32,); declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, int w, int h); for (int w = 2; w <= 128; w <<= 1) { @@ -422,13 +422,13 @@ } report("blend_h"); } static void check_warp8x8(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 15 * 15,); - ALIGN_STK_32(pixel, c_dst, 8 * 8,); - ALIGN_STK_32(pixel, a_dst, 8 * 8,); + ALIGN_STK_64(pixel, src_buf, 15 * 15,); + ALIGN_STK_64(pixel, c_dst, 8 * 8,); + ALIGN_STK_64(pixel, a_dst, 8 * 8,); int16_t abcd[4]; const pixel *src = src_buf + 15 * 3 + 3; const ptrdiff_t dst_stride = 8 * sizeof(pixel); const ptrdiff_t src_stride = 15 * sizeof(pixel); @@ -460,13 +460,13 @@ } report("warp8x8"); } static void check_warp8x8t(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, src_buf, 15 * 15,); - ALIGN_STK_32(int16_t, c_tmp, 8 * 8,); - ALIGN_STK_32(int16_t, a_tmp, 8 * 8,); + ALIGN_STK_64(pixel, src_buf, 15 * 15,); + ALIGN_STK_64(int16_t, c_tmp, 8 * 8,); + ALIGN_STK_64(int16_t, a_tmp, 8 * 8,); int16_t abcd[4]; const pixel *src = src_buf + 15 * 3 + 3; const ptrdiff_t src_stride = 15 * sizeof(pixel); declare_func(void, int16_t *tmp, ptrdiff_t tmp_stride, const pixel *src, @@ -532,13 +532,13 @@ set_off(LEFT, RIGHT, x, w); set_off(TOP, BOTTOM, y, h); } static void check_emuedge(Dav1dMCDSPContext *const c) { - ALIGN_STK_32(pixel, c_dst, 135 * 192,); - ALIGN_STK_32(pixel, a_dst, 135 * 192,); - ALIGN_STK_32(pixel, src, 160 * 160,); + ALIGN_STK_64(pixel, c_dst, 135 * 192,); + ALIGN_STK_64(pixel, a_dst, 135 * 192,); + ALIGN_STK_64(pixel, src, 160 * 160,); for (int i = 0; i < 160 * 160; i++) src[i] = rnd() & ((1U << BITDEPTH) - 1); declare_func(void, intptr_t bw, intptr_t bh, intptr_t iw, intptr_t ih, @@ -571,10 +571,64 @@ } } report("emu_edge"); } +static int get_upscale_x0(const int in_w, const int out_w, const int step) { + const int err = out_w * step - (in_w << 14); + const int x0 = (-((out_w - in_w) << 13) + (out_w >> 1)) / out_w + 128 - (err >> 1); + return x0 & 0x3fff; +} + +static void check_resize(Dav1dMCDSPContext *const c) { + ALIGN_STK_64(pixel, c_dst, 1024 * 64,); + ALIGN_STK_64(pixel, a_dst, 1024 * 64,); + ALIGN_STK_64(pixel, src, 512 * 64,); + + const int height = 64; + const int max_src_width = 512; + const ptrdiff_t dst_stride = 1024 * sizeof(pixel); + const ptrdiff_t src_stride = 512 * sizeof(pixel); + + declare_func(void, pixel *dst, ptrdiff_t dst_stride, + const pixel *src, ptrdiff_t src_stride, + int dst_w, int src_w, int h, int dx, int mx0 + HIGHBD_DECL_SUFFIX); + + if (check_func(c->resize, "resize_%dbpc", BITDEPTH)) { +#if BITDEPTH == 16 + const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; +#else + const int bitdepth_max = 0xff; +#endif + + for (int i = 0; i < max_src_width * height; i++) + src[i] = rnd() & bitdepth_max; + + const int w_den = 9 + (rnd() & 7); + const int src_w = 16 + (rnd() % (max_src_width - 16 + 1)); + const int dst_w = w_den * src_w >> 3; +#define scale_fac(ref_sz, this_sz) \ + ((((ref_sz) << 14) + ((this_sz) >> 1)) / (this_sz)) + const int dx = scale_fac(src_w, dst_w); +#undef scale_fac + const int mx0 = get_upscale_x0(src_w, dst_w, dx); + + call_ref(c_dst, dst_stride, src, src_stride, + dst_w, height, src_w, dx, mx0 HIGHBD_TAIL_SUFFIX); + call_new(a_dst, dst_stride, src, src_stride, + dst_w, height, src_w, dx, mx0 HIGHBD_TAIL_SUFFIX); + checkasm_check_pixel(c_dst, dst_stride, a_dst, dst_stride, + dst_w, height, "dst"); + + bench_new(a_dst, dst_stride, src, src_stride, + 512, height, 512 * 8 / w_den, dx, mx0 HIGHBD_TAIL_SUFFIX); + } + + report("resize"); +} + void bitfn(checkasm_check_mc)(void) { Dav1dMCDSPContext c; bitfn(dav1d_mc_dsp_init)(&c); check_mc(&c); @@ -587,6 +641,7 @@ check_blend_v(&c); check_blend_h(&c); check_warp8x8(&c); check_warp8x8t(&c); check_emuedge(&c); + check_resize(&c); } diff --git a/third_party/dav1d/tests/checkasm/msac.c b/third_party/dav1d/tests/checkasm/msac.c --- a/third_party/dav1d/tests/checkasm/msac.c +++ b/third_party/dav1d/tests/checkasm/msac.c @@ -256,10 +256,16 @@ c.bool_adapt = dav1d_msac_decode_bool_adapt_sse2; c.bool_equi = dav1d_msac_decode_bool_equi_sse2; c.bool = dav1d_msac_decode_bool_sse2; c.hi_tok = dav1d_msac_decode_hi_tok_sse2; } + +#if ARCH_X86_64 + if (dav1d_get_cpu_flags() & DAV1D_X86_CPU_FLAG_AVX2) { + c.symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_avx2; + } +#endif #endif uint8_t buf[BUF_SIZE]; for (int i = 0; i < BUF_SIZE; i++) buf[i] = rnd(); diff --git a/third_party/dav1d/tests/checkasm/x86/checkasm.asm b/third_party/dav1d/tests/checkasm/x86/checkasm.asm --- a/third_party/dav1d/tests/checkasm/x86/checkasm.asm +++ b/third_party/dav1d/tests/checkasm/x86/checkasm.asm @@ -168,10 +168,23 @@ mov rdx, r10 mov rax, r9 .ok: RET +; trigger a warmup of vector units +%macro WARMUP 0 +cglobal warmup, 0, 0 + xorps m0, m0 + mulps m0, m0 + RET +%endmacro + +INIT_YMM avx2 +WARMUP +INIT_ZMM avx512 +WARMUP + %else ; just random numbers to reduce the chance of incidental match %define n3 dword 0x6549315c %define n4 dword 0xe02f3e23 diff --git a/third_party/dav1d/tests/libfuzzer/meson.build b/third_party/dav1d/tests/libfuzzer/meson.build new file mode 100644 --- /dev/null +++ b/third_party/dav1d/tests/libfuzzer/meson.build @@ -0,0 +1,102 @@ +# Copyright © 2020, VideoLAN and dav1d authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# Build definition for the dav1d fuzzing binaries +# + +if fuzzing_engine == 'none' and not have_fseeko + subdir_done() +endif + +dav1d_fuzzer_sources = files('dav1d_fuzzer.c') +fuzzer_ldflags = [] +fuzzer_link_lang = {} + +if get_option('fuzzer_ldflags') != '' + fuzzer_ldflags += [get_option('fuzzer_ldflags')] +endif + +if fuzzing_engine == 'none' + dav1d_fuzzer_sources += files('main.c') +elif fuzzing_engine == 'libfuzzer' + fuzzer_ldflags += ['-fsanitize=fuzzer'] +elif fuzzing_engine == 'oss-fuzz' + # libFuzzingEngine needs c++ + add_languages('cpp') + fuzzer_link_lang = {'link_language': 'cpp'} +endif + +dav1d_fuzzer = executable('dav1d_fuzzer', + dav1d_fuzzer_sources, + include_directories: dav1d_inc_dirs, + c_args: [stackalign_flag, stackrealign_flag], + link_args: fuzzer_ldflags, + link_with : libdav1d, + build_by_default: true, + dependencies : [thread_dependency], + kwargs: fuzzer_link_lang + ) + +dav1d_fuzzer_mt = executable('dav1d_fuzzer_mt', + dav1d_fuzzer_sources, + include_directories: dav1d_inc_dirs, + c_args: [stackalign_flag, stackrealign_flag, '-DDAV1D_MT_FUZZING'], + link_args: fuzzer_ldflags, + link_with : libdav1d, + build_by_default: true, + dependencies : [thread_dependency], + kwargs: fuzzer_link_lang + ) + +objcopy = find_program('objcopy', + required: false) +if (objcopy.found() and + not get_option('b_lto') and + get_option('default_library') == 'static' and + cc.has_function('posix_memalign', prefix : '#include ', args : test_args)) + + libdav1d_af = custom_target('libdav1d_af', + input: libdav1d, + output: 'libdav1d_af.a', + depends: libdav1d, + command: [objcopy, + '--redefine-sym', 'malloc=__wrap_malloc', + '--redefine-sym', 'posix_memalign=__wrap_posix_memalign', + '--redefine-sym', 'pthread_create=__wrap_pthread_create', + '--redefine-sym', 'pthread_cond_init=__wrap_pthread_cond_init', + '--redefine-sym', 'pthread_mutex_init=__wrap_pthread_mutex_init', + '@INPUT@', '@OUTPUT@']) + + dav1d_fuzzer_mem = executable('dav1d_fuzzer_mem', + dav1d_fuzzer_sources + ['alloc_fail.c'], + include_directories: dav1d_inc_dirs, + c_args: [stackalign_flag, stackrealign_flag, '-DDAV1D_ALLOC_FAIL'], + link_args: fuzzer_ldflags + [join_paths(libdav1d_af.full_path())], + link_depends: libdav1d_af, + build_by_default: false, + dependencies : [thread_dependency], + kwargs: fuzzer_link_lang + ) +endif diff --git a/third_party/dav1d/tests/meson.build b/third_party/dav1d/tests/meson.build --- a/third_party/dav1d/tests/meson.build +++ b/third_party/dav1d/tests/meson.build @@ -88,80 +88,24 @@ ], include_directories: dav1d_inc_dirs, c_args: [stackalign_flag, stackrealign_flag], build_by_default: false, - dependencies : [thread_dependency, rt_dependency, m_lib], + dependencies : [ + thread_dependency, + rt_dependency, + libdl_dependency, + m_lib, + ], ) test('checkasm', checkasm, is_parallel: false) endif -dav1d_fuzzer_sources = files('libfuzzer/dav1d_fuzzer.c') -fuzzer_ldflags = [] - -if get_option('fuzzer_ldflags') != '' - fuzzer_ldflags += [get_option('fuzzer_ldflags')] -endif - -if fuzzing_engine == 'none' - dav1d_fuzzer_sources += files('libfuzzer/main.c') -elif fuzzing_engine == 'libfuzzer' - fuzzer_ldflags += ['-fsanitize=fuzzer'] -elif fuzzing_engine == 'oss-fuzz' - # libFuzzingEngine needs libc++ - fuzzer_ldflags += ['-lc++'] -endif - -dav1d_fuzzer = executable('dav1d_fuzzer', - dav1d_fuzzer_sources, - include_directories: dav1d_inc_dirs, - c_args: [stackalign_flag, stackrealign_flag], - link_args: fuzzer_ldflags, - link_with : libdav1d, - build_by_default: true, - dependencies : [thread_dependency], - ) - -dav1d_fuzzer_mt = executable('dav1d_fuzzer_mt', - dav1d_fuzzer_sources, - include_directories: dav1d_inc_dirs, - c_args: [stackalign_flag, stackrealign_flag, '-DDAV1D_MT_FUZZING'], - link_args: fuzzer_ldflags, - link_with : libdav1d, - build_by_default: true, - dependencies : [thread_dependency], - ) - -objcopy = find_program('objcopy', - required: false) -if (objcopy.found() and - not get_option('b_lto') and - get_option('default_library') == 'static' and - cc.has_function('posix_memalign', prefix : '#include ', args : test_args)) - - libdav1d_af = custom_target('libdav1d_af', - input: libdav1d, - output: 'libdav1d_af.a', - depends: libdav1d, - command: [objcopy, - '--redefine-sym', 'malloc=__wrap_malloc', - '--redefine-sym', 'posix_memalign=__wrap_posix_memalign', - '--redefine-sym', 'pthread_create=__wrap_pthread_create', - '--redefine-sym', 'pthread_cond_init=__wrap_pthread_cond_init', - '--redefine-sym', 'pthread_mutex_init=__wrap_pthread_mutex_init', - '@INPUT@', '@OUTPUT@']) - - dav1d_fuzzer_mem = executable('dav1d_fuzzer_mem', - dav1d_fuzzer_sources + ['libfuzzer/alloc_fail.c'], - include_directories: dav1d_inc_dirs, - c_args: [stackalign_flag, stackrealign_flag, '-DDAV1D_ALLOC_FAIL'], - link_args: fuzzer_ldflags + [join_paths(libdav1d_af.full_path())], - link_depends: libdav1d_af, - build_by_default: false, - dependencies : [thread_dependency], - ) +# fuzzing binaries +if meson.version().version_compare('>=0.49') + subdir('libfuzzer') endif # Include dav1d test data repository with additional tests if get_option('testdata_tests') subdir('dav1d-test-data') diff --git a/third_party/dav1d/tools/dav1d.c b/third_party/dav1d/tools/dav1d.c --- a/third_party/dav1d/tools/dav1d.c +++ b/third_party/dav1d/tools/dav1d.c @@ -111,22 +111,28 @@ } static void print_stats(const int istty, const unsigned n, const unsigned num, const uint64_t elapsed, const double i_fps) { - if (istty) fputs("\r", stderr); - const double d_fps = 1e9 * n / elapsed; - const double speed = d_fps / i_fps; - if (num == 0xFFFFFFFF) { - fprintf(stderr, "Decoded %u frames", n); - } else { - fprintf(stderr, "Decoded %u/%u frames (%.1lf%%)", n, num, - 100.0 * n / num); + char buf[80], *b = buf, *const end = buf + 80; + + if (istty) + *b++ = '\r'; + if (num == 0xFFFFFFFF) + b += snprintf(b, end - b, "Decoded %u frames", n); + else + b += snprintf(b, end - b, "Decoded %u/%u frames (%.1lf%%)", + n, num, 100.0 * n / num); + if (i_fps && b < end) { + const double d_fps = 1e9 * n / elapsed; + const double speed = d_fps / i_fps; + b += snprintf(b, end - b, " - %.2lf/%.2lf fps (%.2lfx)", + d_fps, i_fps, speed); } - if (i_fps) - fprintf(stderr, " - %.2lf/%.2lf fps (%.2lfx)", d_fps, i_fps, speed); - if (!istty) fputs("\n", stderr); + if (!istty) + strcpy(b > end - 2 ? end - 2 : b, "\n"); + fputs(buf, stderr); } int main(const int argc, char *const *const argv) { const int istty = isatty(fileno(stderr)); int res = 0; @@ -147,12 +153,10 @@ fprintf(stderr, "Version mismatch (library: %s, executable: %s)\n", version, DAV1D_VERSION); return EXIT_FAILURE; } - init_demuxers(); - init_muxers(); parse(argc, argv, &cli_settings, &lib_settings); if ((res = input_open(&in, cli_settings.demuxer, cli_settings.inputfile, fps, &total, timebase)) < 0) diff --git a/third_party/dav1d/tools/dav1d_cli_parse.c b/third_party/dav1d/tools/dav1d_cli_parse.c --- a/third_party/dav1d/tools/dav1d_cli_parse.c +++ b/third_party/dav1d/tools/dav1d_cli_parse.c @@ -84,11 +84,11 @@ #if ARCH_AARCH64 || ARCH_ARM #define ALLOWED_CPU_MASKS " or 'neon'" #elif ARCH_X86 #define ALLOWED_CPU_MASKS \ - ", 'sse2', 'ssse3', 'sse41', 'avx2' or 'avx512'" + ", 'sse2', 'ssse3', 'sse41', 'avx2' or 'avx512icl'" #else #define ALLOWED_CPU_MASKS "not yet implemented for this architecture" #endif static void usage(const char *const app, const char *const reason, ...) { @@ -100,11 +100,11 @@ va_end(args); fprintf(stderr, "\n\n"); } fprintf(stderr, "Usage: %s [options]\n\n", app); fprintf(stderr, "Supported options:\n" - " --input/-i $file: input file\n" + " --input/-i $file: input file\n" " --output/-o $file: output file\n" " --demuxer $name: force demuxer type ('ivf', 'section5' or 'annexb'; default: detect from extension)\n" " --muxer $name: force muxer type ('md5', 'yuv', 'yuv4mpeg2' or 'null'; default: detect from extension)\n" " --quiet/-q: disable status messages\n" " --frametimes $file: dump frame times to file\n" @@ -174,31 +174,27 @@ const int val; } EnumParseTable; #if ARCH_X86 enum CpuMask { - X86_CPU_MASK_SSE = DAV1D_X86_CPU_FLAG_SSE, - X86_CPU_MASK_SSE2 = DAV1D_X86_CPU_FLAG_SSE2 | X86_CPU_MASK_SSE, - X86_CPU_MASK_SSE3 = DAV1D_X86_CPU_FLAG_SSE3 | X86_CPU_MASK_SSE2, - X86_CPU_MASK_SSSE3 = DAV1D_X86_CPU_FLAG_SSSE3 | X86_CPU_MASK_SSE3, - X86_CPU_MASK_SSE41 = DAV1D_X86_CPU_FLAG_SSE41 | X86_CPU_MASK_SSSE3, - X86_CPU_MASK_SSE42 = DAV1D_X86_CPU_FLAG_SSE42 | X86_CPU_MASK_SSE41, - X86_CPU_MASK_AVX = DAV1D_X86_CPU_FLAG_AVX | X86_CPU_MASK_SSE42, - X86_CPU_MASK_AVX2 = DAV1D_X86_CPU_FLAG_AVX2 | X86_CPU_MASK_AVX, - X86_CPU_MASK_AVX512 = DAV1D_X86_CPU_FLAG_AVX512 | X86_CPU_MASK_AVX2, + X86_CPU_MASK_SSE2 = DAV1D_X86_CPU_FLAG_SSE2, + X86_CPU_MASK_SSSE3 = DAV1D_X86_CPU_FLAG_SSSE3 | X86_CPU_MASK_SSE2, + X86_CPU_MASK_SSE41 = DAV1D_X86_CPU_FLAG_SSE41 | X86_CPU_MASK_SSSE3, + X86_CPU_MASK_AVX2 = DAV1D_X86_CPU_FLAG_AVX2 | X86_CPU_MASK_SSE41, + X86_CPU_MASK_AVX512ICL = DAV1D_X86_CPU_FLAG_AVX512ICL | X86_CPU_MASK_AVX2, }; #endif static const EnumParseTable cpu_mask_tbl[] = { #if ARCH_AARCH64 || ARCH_ARM { "neon", DAV1D_ARM_CPU_FLAG_NEON }, #elif ARCH_X86 - { "sse2", X86_CPU_MASK_SSE2 }, - { "ssse3", X86_CPU_MASK_SSSE3 }, - { "sse41", X86_CPU_MASK_SSE41 }, - { "avx2", X86_CPU_MASK_AVX2 }, - { "avx512", X86_CPU_MASK_AVX512 }, + { "sse2", X86_CPU_MASK_SSE2 }, + { "ssse3", X86_CPU_MASK_SSSE3 }, + { "sse41", X86_CPU_MASK_SSE41 }, + { "avx2", X86_CPU_MASK_AVX2 }, + { "avx512icl", X86_CPU_MASK_AVX512ICL }, #endif { 0 }, }; static unsigned parse_enum(char *optarg, const EnumParseTable *const tbl, diff --git a/third_party/dav1d/tools/input/input.h b/third_party/dav1d/tools/input/input.h --- a/third_party/dav1d/tools/input/input.h +++ b/third_party/dav1d/tools/input/input.h @@ -30,11 +30,10 @@ #include "data.h" typedef struct DemuxerContext DemuxerContext; -void init_demuxers(void); int input_open(DemuxerContext **const c_out, const char *const name, const char *const filename, unsigned fps[2], unsigned *num_frames, unsigned timebase[2]); int input_read(DemuxerContext *ctx, Dav1dData *data); void input_close(DemuxerContext *ctx); diff --git a/third_party/dav1d/tools/input/input.c b/third_party/dav1d/tools/input/input.c --- a/third_party/dav1d/tools/input/input.c +++ b/third_party/dav1d/tools/input/input.c @@ -41,48 +41,42 @@ struct DemuxerContext { DemuxerPriv *data; const Demuxer *impl; }; -#define MAX_NUM_DEMUXERS 3 -static const Demuxer *demuxers[MAX_NUM_DEMUXERS]; -static int num_demuxers = 0; - -#define register_demuxer(impl) { \ - extern const Demuxer impl; \ - assert(num_demuxers < MAX_NUM_DEMUXERS); \ - demuxers[num_demuxers++] = &impl; \ -} - -void init_demuxers(void) { - register_demuxer(ivf_demuxer); - register_demuxer(annexb_demuxer); - register_demuxer(section5_demuxer); -} +extern const Demuxer ivf_demuxer; +extern const Demuxer annexb_demuxer; +extern const Demuxer section5_demuxer; +static const Demuxer *const demuxers[] = { + &ivf_demuxer, + &annexb_demuxer, + §ion5_demuxer, + NULL +}; int input_open(DemuxerContext **const c_out, const char *const name, const char *const filename, unsigned fps[2], unsigned *const num_frames, unsigned timebase[2]) { const Demuxer *impl; DemuxerContext *c; int res, i; if (name) { - for (i = 0; i < num_demuxers; i++) { + for (i = 0; demuxers[i]; i++) { if (!strcmp(demuxers[i]->name, name)) { impl = demuxers[i]; break; } } - if (i == num_demuxers) { + if (!demuxers[i]) { fprintf(stderr, "Failed to find demuxer named \"%s\"\n", name); return DAV1D_ERR(ENOPROTOOPT); } } else { int probe_sz = 0; - for (i = 0; i < num_demuxers; i++) + for (i = 0; demuxers[i]; i++) probe_sz = imax(probe_sz, demuxers[i]->probe_sz); uint8_t *const probe_data = malloc(probe_sz); if (!probe_data) { fprintf(stderr, "Failed to allocate memory\n"); return DAV1D_ERR(ENOMEM); @@ -94,30 +88,29 @@ free(probe_data); fprintf(stderr, "Failed to read probe data\n"); return errno ? DAV1D_ERR(errno) : DAV1D_ERR(EIO); } - for (i = 0; i < num_demuxers; i++) { + for (i = 0; demuxers[i]; i++) { if (demuxers[i]->probe(probe_data)) { impl = demuxers[i]; break; } } free(probe_data); - if (i == num_demuxers) { + if (!demuxers[i]) { fprintf(stderr, "Failed to probe demuxer for file %s\n", filename); return DAV1D_ERR(ENOPROTOOPT); } } - if (!(c = malloc(sizeof(DemuxerContext) + impl->priv_data_size))) { + if (!(c = calloc(1, sizeof(DemuxerContext) + impl->priv_data_size))) { fprintf(stderr, "Failed to allocate memory\n"); return DAV1D_ERR(ENOMEM); } - memset(c, 0, sizeof(DemuxerContext) + impl->priv_data_size); c->impl = impl; c->data = (DemuxerPriv *) &c[1]; if ((res = impl->open(c->data, filename, fps, num_frames, timebase)) < 0) { free(c); return res; diff --git a/third_party/dav1d/tools/input/ivf.c b/third_party/dav1d/tools/input/ivf.c --- a/third_party/dav1d/tools/input/ivf.c +++ b/third_party/dav1d/tools/input/ivf.c @@ -61,11 +61,10 @@ unsigned fps[2], unsigned *const num_frames, unsigned timebase[2]) { size_t res; uint8_t hdr[32]; - memset(c, 0, sizeof(*c)); if (!(c->f = fopen(file, "rb"))) { fprintf(stderr, "Failed to open %s: %s\n", file, strerror(errno)); return -1; } else if ((res = fread(hdr, 32, 1, c->f)) != 1) { fprintf(stderr, "Failed to read stream header: %s\n", strerror(errno)); diff --git a/third_party/dav1d/tools/output/output.h b/third_party/dav1d/tools/output/output.h --- a/third_party/dav1d/tools/output/output.h +++ b/third_party/dav1d/tools/output/output.h @@ -30,11 +30,10 @@ #include "picture.h" typedef struct MuxerContext MuxerContext; -void init_muxers(void); int output_open(MuxerContext **c, const char *name, const char *filename, const Dav1dPictureParameters *p, const unsigned fps[2]); int output_write(MuxerContext *ctx, Dav1dPicture *pic); void output_close(MuxerContext *ctx); /** diff --git a/third_party/dav1d/tools/output/output.c b/third_party/dav1d/tools/output/output.c --- a/third_party/dav1d/tools/output/output.c +++ b/third_party/dav1d/tools/output/output.c @@ -40,26 +40,21 @@ struct MuxerContext { MuxerPriv *data; const Muxer *impl; }; -#define MAX_NUM_MUXERS 4 -static const Muxer *muxers[MAX_NUM_MUXERS]; -static unsigned num_muxers = 0; - -#define register_muxer(impl) { \ - extern const Muxer impl; \ - assert(num_muxers < MAX_NUM_MUXERS); \ - muxers[num_muxers++] = &impl; \ -} - -void init_muxers(void) { - register_muxer(null_muxer); - register_muxer(md5_muxer); - register_muxer(yuv_muxer); - register_muxer(y4m2_muxer); -} +extern const Muxer null_muxer; +extern const Muxer md5_muxer; +extern const Muxer yuv_muxer; +extern const Muxer y4m2_muxer; +static const Muxer *muxers[] = { + &null_muxer, + &md5_muxer, + &yuv_muxer, + &y4m2_muxer, + NULL +}; static const char *find_extension(const char *const f) { const size_t l = strlen(f); if (l == 0) return NULL; @@ -84,35 +79,35 @@ MuxerContext *c; unsigned i; int res; if (name) { - for (i = 0; i < num_muxers; i++) { + for (i = 0; muxers[i]; i++) { if (!strcmp(muxers[i]->name, name)) { impl = muxers[i]; break; } } - if (i == num_muxers) { + if (!muxers[i]) { fprintf(stderr, "Failed to find muxer named \"%s\"\n", name); return DAV1D_ERR(ENOPROTOOPT); } } else if (!strcmp(filename, "/dev/null")) { impl = muxers[0]; } else { - const char *ext = find_extension(filename); + const char *const ext = find_extension(filename); if (!ext) { fprintf(stderr, "No extension found for file %s\n", filename); return -1; } - for (i = 0; i < num_muxers; i++) { + for (i = 0; muxers[i]; i++) { if (!strcmp(muxers[i]->extension, ext)) { impl = muxers[i]; break; } } - if (i == num_muxers) { + if (!muxers[i]) { fprintf(stderr, "Failed to find muxer for extension \"%s\"\n", ext); return DAV1D_ERR(ENOPROTOOPT); } } @@ -130,26 +125,21 @@ return 0; } int output_write(MuxerContext *const ctx, Dav1dPicture *const p) { - int res; - - if ((res = ctx->impl->write_picture(ctx->data, p)) < 0) - return res; - - return 0; + const int res = ctx->impl->write_picture(ctx->data, p); + return res < 0 ? res : 0; } void output_close(MuxerContext *const ctx) { if (ctx->impl->write_trailer) ctx->impl->write_trailer(ctx->data); free(ctx); } int output_verify(MuxerContext *const ctx, const char *const md5_str) { - int res = 0; - if (ctx->impl->verify) - res = ctx->impl->verify(ctx->data, md5_str); + const int res = ctx->impl->verify ? + ctx->impl->verify(ctx->data, md5_str) : 0; free(ctx); return res; }