diff --git a/media/libvpx/config/vpx_version.h b/media/libvpx/config/vpx_version.h --- a/media/libvpx/config/vpx_version.h +++ b/media/libvpx/config/vpx_version.h @@ -1,8 +1,8 @@ // This file is generated. Do not edit. #define VERSION_MAJOR 1 -#define VERSION_MINOR 13 -#define VERSION_PATCH 1 +#define VERSION_MINOR 14 +#define VERSION_PATCH 0 #define VERSION_EXTRA "" #define VERSION_PACKED ((VERSION_MAJOR<<16)|(VERSION_MINOR<<8)|(VERSION_PATCH)) -#define VERSION_STRING_NOSP "v1.13.1" -#define VERSION_STRING " v1.13.1" +#define VERSION_STRING_NOSP "v1.14.0" +#define VERSION_STRING " v1.14.0" diff --git a/media/libvpx/libvpx/CHANGELOG b/media/libvpx/libvpx/CHANGELOG --- a/media/libvpx/libvpx/CHANGELOG +++ b/media/libvpx/libvpx/CHANGELOG @@ -1,5 +1,10 @@ +20yy-mm-dd v1.14.0 "V Duck" + This release drops support for old C compilers, such as Visual Studio 2012 + and older, that disallow mixing variable declarations and statements (a C99 + feature). + 2023-09-29 v1.13.1 "Ugly Duckling" This release contains two security related fixes. One each for VP8 and VP9. - Upgrading: This release is ABI compatible with the previous release. diff --git a/media/libvpx/libvpx/build/make/configure.sh b/media/libvpx/libvpx/build/make/configure.sh --- a/media/libvpx/libvpx/build/make/configure.sh +++ b/media/libvpx/libvpx/build/make/configure.sh @@ -861,12 +861,18 @@ soft_enable lasx enable_feature loongarch ;; esac - # PIC is probably what we want when building shared libs + # Position independent code (PIC) is probably what we want when building + # shared libs or position independent executable (PIE) targets. enabled shared && soft_enable pic + check_cpp << EOF || soft_enable pic +#if !(__pie__ || __PIE__) +#error Neither __pie__ or __PIE__ are set +#endif +EOF # Minimum iOS version for all target platforms (darwin and iphonesimulator). # Shared library framework builds are only possible on iOS 8 and later. if enabled shared; then IOS_VERSION_OPTIONS="--enable-shared" diff --git a/media/libvpx/libvpx/configure b/media/libvpx/libvpx/configure --- a/media/libvpx/libvpx/configure +++ b/media/libvpx/libvpx/configure @@ -641,11 +641,10 @@ # Enable some useful compiler flags if enabled gcc; then enabled werror && check_add_cflags -Werror check_add_cflags -Wall - check_add_cflags -Wdeclaration-after-statement check_add_cflags -Wdisabled-optimization check_add_cflags -Wextra-semi check_add_cflags -Wextra-semi-stmt check_add_cflags -Wfloat-conversion check_add_cflags -Wformat=2 @@ -668,13 +667,12 @@ check_cflags -Wframe-larger-than=52000 && \ add_cflags_only -Wframe-larger-than=52000 if enabled mips || [ -z "${INLINE}" ]; then enabled extra_warnings || check_add_cflags -Wno-unused-function fi - # Enforce c89 for c files. Don't be too strict about it though. Allow - # gnu extensions like "//" for comments. - check_cflags -std=gnu89 && add_cflags_only -std=gnu89 + # Enforce C99 for C files. Allow GNU extensions. + check_cflags -std=gnu99 && add_cflags_only -std=gnu99 # Avoid this warning for third_party C++ sources. Some reorganization # would be needed to apply this only to test/*.cc. check_cflags -Wshorten-64-to-32 && add_cflags_only -Wshorten-64-to-32 # Quiet gcc 6 vs 7 abi warnings: diff --git a/media/libvpx/libvpx/test/encode_api_test.cc b/media/libvpx/libvpx/test/encode_api_test.cc --- a/media/libvpx/libvpx/test/encode_api_test.cc +++ b/media/libvpx/libvpx/test/encode_api_test.cc @@ -19,10 +19,13 @@ #include "test/i420_video_source.h" #include "test/video_source.h" #include "./vpx_config.h" #include "vpx/vp8cx.h" +#include "vpx/vpx_codec.h" +#include "vpx/vpx_encoder.h" +#include "vpx/vpx_image.h" #include "vpx/vpx_tpl.h" namespace { const vpx_codec_iface_t *kCodecIfaces[] = { @@ -120,10 +123,66 @@ free(img_buf); vpx_codec_destroy(&enc); } + +// Verifies the fix for a float-cast-overflow in vp8_change_config(). +// +// Causes cpi->framerate to become the largest possible value (10,000,000) in +// VP8 by setting cfg.g_timebase to 1/10000000 and passing a duration of 1 to +// vpx_codec_encode(). +TEST(EncodeAPI, HugeFramerateVp8) { + vpx_codec_iface_t *const iface = vpx_codec_vp8_cx(); + vpx_codec_enc_cfg_t cfg; + ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK); + cfg.g_w = 271; + cfg.g_h = 1080; + cfg.g_timebase.num = 1; + // Largest value (VP8's TICKS_PER_SEC) such that frame duration is nonzero (1 + // tick). + cfg.g_timebase.den = 10000000; + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_lag_in_frames = 0; + cfg.rc_end_usage = VPX_CBR; + + vpx_codec_ctx_t enc; + // Before we encode the first frame, cpi->framerate is set to a guess (the + // reciprocal of cfg.g_timebase). If this guess doesn't seem reasonable + // (> 180), cpi->framerate is set to 30. + ASSERT_EQ(vpx_codec_enc_init(&enc, iface, &cfg, 0), VPX_CODEC_OK); + + ASSERT_EQ(vpx_codec_control(&enc, VP8E_SET_CPUUSED, -12), VPX_CODEC_OK); + + vpx_image_t *const image = + vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, cfg.g_w, cfg.g_h, 1); + ASSERT_NE(image, nullptr); + + for (unsigned int i = 0; i < image->d_h; ++i) { + memset(image->planes[0] + i * image->stride[0], 128, image->d_w); + } + const unsigned int uv_h = (image->d_h + 1) / 2; + const unsigned int uv_w = (image->d_w + 1) / 2; + for (unsigned int i = 0; i < uv_h; ++i) { + memset(image->planes[1] + i * image->stride[1], 128, uv_w); + memset(image->planes[2] + i * image->stride[2], 128, uv_w); + } + + // Encode a frame. + // Up to this point cpi->framerate is 30. Now pass a duration of only 1. This + // causes cpi->framerate to become 10,000,000. + ASSERT_EQ(vpx_codec_encode(&enc, image, 0, 1, 0, VPX_DL_REALTIME), + VPX_CODEC_OK); + + // Change to the same config. Since cpi->framerate is now huge, when it is + // used to calculate raw_target_rate (bit rate of uncompressed frames), the + // result is likely to overflow an unsigned int. + ASSERT_EQ(vpx_codec_enc_config_set(&enc, &cfg), VPX_CODEC_OK); + + vpx_img_free(image); + ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); +} #endif // Set up 2 spatial streams with 2 temporal layers per stream, and generate // invalid configuration by setting the temporal layer rate allocation // (ts_target_bitrate[]) to 0 for both layers. This should fail independent of @@ -435,10 +494,181 @@ << "target bitrate: " << cfg.rc_target_bitrate << " framerate: " << static_cast(cfg.g_timebase.den) / cfg.g_timebase.num; } #endif // VPX_ARCH_X86_64 || VPX_ARCH_AARCH64 +vpx_image_t *CreateImage(const unsigned int width, const unsigned int height) { + vpx_image_t *image = + vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, width, height, 1); + if (!image) return image; + + for (unsigned int i = 0; i < image->d_h; ++i) { + memset(image->planes[0] + i * image->stride[0], 128, image->d_w); + } + const unsigned int uv_h = (image->d_h + 1) / 2; + const unsigned int uv_w = (image->d_w + 1) / 2; + for (unsigned int i = 0; i < uv_h; ++i) { + memset(image->planes[1] + i * image->stride[1], 128, uv_w); + memset(image->planes[2] + i * image->stride[2], 128, uv_w); + } + + return image; +} + +void CodecEncodeFrame(vpx_codec_ctx_t *enc, const int width, const int height, + const int frame_index, unsigned int deadline, + const bool is_key) { + const vpx_codec_cx_pkt_t *pkt; + vpx_image_t *image = CreateImage(width, height); + vpx_enc_frame_flags_t frame_flags = is_key ? VPX_EFLAG_FORCE_KF : 0; + ASSERT_NE(image, nullptr); + ASSERT_EQ(vpx_codec_encode(enc, image, frame_index, 1, frame_flags, deadline), + VPX_CODEC_OK); + vpx_codec_iter_t iter = nullptr; + while ((pkt = vpx_codec_get_cx_data(enc, &iter)) != nullptr) { + ASSERT_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT); + } + vpx_img_free(image); +} + +// This is a test case from clusterfuzz. +TEST(EncodeAPI, PrevMiCheckNullptr) { + vpx_codec_iface_t *const iface = vpx_codec_vp9_cx(); + vpx_codec_enc_cfg_t cfg; + + struct Config { + unsigned int thread; + unsigned int width; + unsigned int height; + vpx_rc_mode end_usage; + unsigned long deadline; + }; + struct Config init_config = { 0, 1554, 644, VPX_VBR, 1 }; + unsigned long deadline = init_config.deadline; + ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, /*usage=*/0), + VPX_CODEC_OK); + cfg.g_threads = init_config.thread; + cfg.g_w = init_config.width; + cfg.g_h = init_config.height; + cfg.g_timebase.num = 1; + cfg.g_timebase.den = 1000 * 1000; // microseconds + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_lag_in_frames = 0; + cfg.rc_end_usage = init_config.end_usage; + cfg.rc_min_quantizer = 2; + cfg.rc_max_quantizer = 58; + + vpx_codec_ctx_t enc; + ASSERT_EQ(vpx_codec_enc_init(&enc, iface, &cfg, 0), VPX_CODEC_OK); + ASSERT_EQ(vpx_codec_control(&enc, VP8E_SET_CPUUSED, 0), VPX_CODEC_OK); + + const vpx_codec_cx_pkt_t *pkt; + + int frame_index = 0; + // First step: encode, without forcing KF. + vpx_image_t *image = CreateImage(cfg.g_w, cfg.g_h); + ASSERT_NE(image, nullptr); + ASSERT_EQ(vpx_codec_encode(&enc, image, frame_index, 1, 0, deadline), + VPX_CODEC_OK); + frame_index++; + vpx_codec_iter_t iter = nullptr; + while ((pkt = vpx_codec_get_cx_data(&enc, &iter)) != nullptr) { + ASSERT_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT); + } + vpx_img_free(image); + // Second step: change config + struct Config encode_config = { 0, 1131, 644, VPX_CBR, 1000000 }; + cfg.g_threads = encode_config.thread; + cfg.g_w = encode_config.width; + cfg.g_h = encode_config.height; + cfg.rc_end_usage = encode_config.end_usage; + deadline = encode_config.deadline; + ASSERT_EQ(vpx_codec_enc_config_set(&enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(&enc); + // Third step: encode, without forcing KF + image = CreateImage(cfg.g_w, cfg.g_h); + ASSERT_NE(image, nullptr); + ASSERT_EQ(vpx_codec_encode(&enc, image, frame_index, 1, 0, deadline), + VPX_CODEC_OK); + frame_index++; + while ((pkt = vpx_codec_get_cx_data(&enc, &iter)) != nullptr) { + ASSERT_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT); + } + vpx_img_free(image); + ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); +} + +// This is a test case from clusterfuzz: based on 310477034. +// Encode a few frames with multiple change config call +// with different frame size. +TEST(EncodeAPI, MultipleChangeConfigResize) { + vpx_codec_iface_t *const iface = vpx_codec_vp9_cx(); + vpx_codec_enc_cfg_t cfg; + + struct Config { + unsigned int thread; + unsigned int width; + unsigned int height; + vpx_rc_mode end_usage; + unsigned long deadline; + }; + + // Set initial config. + struct Config config = { 3, 41, 1, VPX_VBR, 1 }; + unsigned long deadline = config.deadline; + ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, /*usage=*/0), + VPX_CODEC_OK); + cfg.g_threads = config.thread; + cfg.g_w = config.width; + cfg.g_h = config.height; + cfg.g_timebase.num = 1; + cfg.g_timebase.den = 1000 * 1000; // microseconds + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_lag_in_frames = 0; + cfg.rc_end_usage = config.end_usage; + cfg.rc_min_quantizer = 2; + cfg.rc_max_quantizer = 58; + + vpx_codec_ctx_t enc; + ASSERT_EQ(vpx_codec_enc_init(&enc, iface, &cfg, 0), VPX_CODEC_OK); + ASSERT_EQ(vpx_codec_control(&enc, VP8E_SET_CPUUSED, 3), VPX_CODEC_OK); + int frame_index = 0; + + // Encode first frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, true); + frame_index++; + + // Change config. + config = { 16, 31, 1, VPX_VBR, 1000000 }; + cfg.g_threads = config.thread; + cfg.g_w = config.width; + cfg.g_h = config.height; + cfg.rc_end_usage = config.end_usage; + ASSERT_EQ(vpx_codec_enc_config_set(&enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(&enc); + + // Change config again. + config = { 0, 17, 1, VPX_CBR, 1 }; + cfg.g_threads = config.thread; + cfg.g_w = config.width; + cfg.g_h = config.height; + cfg.rc_end_usage = config.end_usage; + deadline = config.deadline; + ASSERT_EQ(vpx_codec_enc_config_set(&enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(&enc); + + // Encode 2nd frame with new config, set delta frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, false); + frame_index++; + + // Encode 3rd frame with same config, set delta frame. + CodecEncodeFrame(&enc, cfg.g_w, cfg.g_h, frame_index, deadline, false); + frame_index++; + + ASSERT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); +} + class EncodeApiGetTplStatsTest : public ::libvpx_test::EncoderTest, public ::testing::TestWithParam { public: EncodeApiGetTplStatsTest() : EncoderTest(GetParam()), test_io_(false) {} diff --git a/media/libvpx/libvpx/test/tools_common.sh b/media/libvpx/libvpx/test/tools_common.sh --- a/media/libvpx/libvpx/test/tools_common.sh +++ b/media/libvpx/libvpx/test/tools_common.sh @@ -278,11 +278,16 @@ "${test}" vlog " PASS ${test}" test_end "${test}" done - local tested_config="$(test_configuration_target) @ $(current_hash)" + # C vs SIMD tests are run for x86 32-bit, 64-bit and ARM platform + if [ "${test_name}" = "vp9_c_vs_simd_encode" ]; then + local tested_config="$(current_hash)" + else + local tested_config="$(test_configuration_target) @ $(current_hash)" + fi echo "${test_name}: Done, all tests pass for ${tested_config}." } vpx_test_usage() { cat << EOF diff --git a/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc --- a/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc +++ b/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc @@ -50,11 +50,12 @@ class Vp8RcInterfaceTest : public ::libvpx_test::EncoderTest, public ::libvpx_test::CodecTestWith2Params { public: Vp8RcInterfaceTest() - : EncoderTest(GET_PARAM(0)), key_interval_(3000), encoder_exit_(false) {} + : EncoderTest(GET_PARAM(0)), key_interval_(3000), encoder_exit_(false), + frame_drop_thresh_(0) {} ~Vp8RcInterfaceTest() override = default; protected: void SetUp() override { InitializeConfig(); @@ -143,23 +144,24 @@ if (encoder_exit_) { return; } int qp; encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); - rc_api_->ComputeQP(frame_params_); - ASSERT_EQ(rc_api_->GetQP(), qp); + if (rc_api_->ComputeQP(frame_params_) == libvpx::FrameDropDecision::kOk) { + ASSERT_EQ(rc_api_->GetQP(), qp); + } else { + num_drops_++; + } } void FramePktHook(const vpx_codec_cx_pkt_t *pkt) override { rc_api_->PostEncodeUpdate(pkt->data.frame.sz); } void RunOneLayer() { test_video_ = GET_PARAM(2); target_bitrate_ = GET_PARAM(1); - if (test_video_.width == 1280 && target_bitrate_ == 200) return; - if (test_video_.width == 640 && target_bitrate_ == 1000) return; SetConfig(); rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); ASSERT_TRUE(rc_api_->UpdateRateControl(rc_cfg_)); ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, @@ -167,16 +169,37 @@ test_video_.frames); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); } + void RunOneLayerDropFrames() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + frame_drop_thresh_ = 30; + num_drops_ = 0; + // Use lower target_bitrate and max_quantizer to trigger drops. + target_bitrate_ = target_bitrate_ >> 2; + SetConfig(); + rc_cfg_.max_quantizer = 56; + cfg_.rc_max_quantizer = 56; + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + ASSERT_TRUE(rc_api_->UpdateRateControl(rc_cfg_)); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Check that some frames were dropped, otherwise test has no value. + ASSERT_GE(num_drops_, 1); + } + void RunPeriodicKey() { test_video_ = GET_PARAM(2); target_bitrate_ = GET_PARAM(1); - if (test_video_.width == 1280 && target_bitrate_ == 200) return; - if (test_video_.width == 640 && target_bitrate_ == 1000) return; key_interval_ = 100; + frame_drop_thresh_ = 30; SetConfig(); rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); ASSERT_TRUE(rc_api_->UpdateRateControl(rc_cfg_)); ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, @@ -187,12 +210,10 @@ } void RunTemporalLayers2TL() { test_video_ = GET_PARAM(2); target_bitrate_ = GET_PARAM(1); - if (test_video_.width == 1280 && target_bitrate_ == 200) return; - if (test_video_.width == 640 && target_bitrate_ == 1000) return; SetConfigTemporalLayers(2); rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); ASSERT_TRUE(rc_api_->UpdateRateControl(rc_cfg_)); ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, @@ -203,12 +224,10 @@ } void RunTemporalLayers3TL() { test_video_ = GET_PARAM(2); target_bitrate_ = GET_PARAM(1); - if (test_video_.width == 1280 && target_bitrate_ == 200) return; - if (test_video_.width == 640 && target_bitrate_ == 1000) return; SetConfigTemporalLayers(3); rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); ASSERT_TRUE(rc_api_->UpdateRateControl(rc_cfg_)); ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, @@ -216,10 +235,32 @@ test_video_.frames); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); } + void RunTemporalLayers3TLDropFrames() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + frame_drop_thresh_ = 30; + num_drops_ = 0; + // Use lower target_bitrate and max_quantizer to trigger drops. + target_bitrate_ = target_bitrate_ >> 2; + SetConfigTemporalLayers(3); + rc_cfg_.max_quantizer = 56; + cfg_.rc_max_quantizer = 56; + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + ASSERT_TRUE(rc_api_->UpdateRateControl(rc_cfg_)); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Check that some frames were dropped, otherwise test has no value. + ASSERT_GE(num_drops_, 1); + } + private: void SetConfig() { rc_cfg_.width = test_video_.width; rc_cfg_.height = test_video_.height; rc_cfg_.max_quantizer = 60; @@ -231,10 +272,11 @@ rc_cfg_.undershoot_pct = 50; rc_cfg_.overshoot_pct = 50; rc_cfg_.max_intra_bitrate_pct = 1000; rc_cfg_.framerate = 30.0; rc_cfg_.layer_target_bitrate[0] = target_bitrate_; + rc_cfg_.frame_drop_thresh = frame_drop_thresh_; // Encoder settings for ground truth. cfg_.g_w = test_video_.width; cfg_.g_h = test_video_.height; cfg_.rc_undershoot_pct = 50; @@ -249,10 +291,11 @@ cfg_.g_lag_in_frames = 0; cfg_.g_error_resilient = 1; cfg_.rc_target_bitrate = target_bitrate_; cfg_.kf_min_dist = key_interval_; cfg_.kf_max_dist = key_interval_; + cfg_.rc_dropframe_thresh = frame_drop_thresh_; } void SetConfigTemporalLayers(int temporal_layers) { rc_cfg_.width = test_video_.width; rc_cfg_.height = test_video_.height; @@ -264,10 +307,11 @@ rc_cfg_.buf_sz = target_bitrate_; rc_cfg_.undershoot_pct = 50; rc_cfg_.overshoot_pct = 50; rc_cfg_.max_intra_bitrate_pct = 1000; rc_cfg_.framerate = 30.0; + rc_cfg_.frame_drop_thresh = frame_drop_thresh_; if (temporal_layers == 2) { rc_cfg_.layer_target_bitrate[0] = 60 * target_bitrate_ / 100; rc_cfg_.layer_target_bitrate[1] = target_bitrate_; rc_cfg_.ts_rate_decimator[0] = 2; rc_cfg_.ts_rate_decimator[1] = 1; @@ -297,10 +341,11 @@ cfg_.g_lag_in_frames = 0; cfg_.g_error_resilient = 1; cfg_.rc_target_bitrate = target_bitrate_; cfg_.kf_min_dist = key_interval_; cfg_.kf_max_dist = key_interval_; + cfg_.rc_dropframe_thresh = frame_drop_thresh_; // 2 Temporal layers, no spatial layers, CBR mode. cfg_.ss_number_layers = 1; cfg_.ts_number_layers = temporal_layers; if (temporal_layers == 2) { cfg_.ts_rate_decimator[0] = 2; @@ -324,20 +369,28 @@ int key_interval_; int target_bitrate_; Vp8RCTestVideo test_video_; libvpx::VP8FrameParamsQpRTC frame_params_; bool encoder_exit_; + int frame_drop_thresh_; + int num_drops_; }; TEST_P(Vp8RcInterfaceTest, OneLayer) { RunOneLayer(); } +TEST_P(Vp8RcInterfaceTest, OneLayerDropFrames) { RunOneLayerDropFrames(); } + TEST_P(Vp8RcInterfaceTest, OneLayerPeriodicKey) { RunPeriodicKey(); } TEST_P(Vp8RcInterfaceTest, TemporalLayers2TL) { RunTemporalLayers2TL(); } TEST_P(Vp8RcInterfaceTest, TemporalLayers3TL) { RunTemporalLayers3TL(); } +TEST_P(Vp8RcInterfaceTest, TemporalLayers3TLDropFrames) { + RunTemporalLayers3TLDropFrames(); +} + VP8_INSTANTIATE_TEST_SUITE(Vp8RcInterfaceTest, ::testing::Values(200, 400, 1000), ::testing::ValuesIn(kVp8RCTestVectors)); } // namespace diff --git a/media/libvpx/libvpx/test/vp9_c_vs_simd_encode.sh b/media/libvpx/libvpx/test/vp9_c_vs_simd_encode.sh --- a/media/libvpx/libvpx/test/vp9_c_vs_simd_encode.sh +++ b/media/libvpx/libvpx/test/vp9_c_vs_simd_encode.sh @@ -382,20 +382,18 @@ vp9_c_vs_simd_enc_test() { # Test Generic vp9_test_generic - # TODO(webm:1816): Enable x86 test once issue 1816 is fixed. - # Details: https://bugs.chromium.org/p/webm/issues/detail?id=1816 # Test x86 (32 bit) - # echo "vp9 test for x86 (32 bit): Started." - # if ! vp9_test_x86 "x86"; then - # echo "vp9 test for x86 (32 bit): Done, test failed." - # return 1 - # else - # echo "vp9 test for x86 (32 bit): Done, all tests passed." - # fi + echo "vp9 test for x86 (32 bit): Started." + if ! vp9_test_x86 "x86"; then + echo "vp9 test for x86 (32 bit): Done, test failed." + return 1 + else + echo "vp9 test for x86 (32 bit): Done, all tests passed." + fi # Test x86_64 (64 bit) if [ "$(eval uname -m)" = "x86_64" ]; then echo "vp9 test for x86_64 (64 bit): Started." if ! vp9_test_x86 "x86_64"; then diff --git a/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc --- a/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc +++ b/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc @@ -29,18 +29,19 @@ const int kTemporalId3Layer[4] = { 0, 2, 1, 2 }; const int kTemporalId2Layer[2] = { 0, 1 }; const int kTemporalRateAllocation3Layer[3] = { 50, 70, 100 }; const int kTemporalRateAllocation2Layer[2] = { 60, 100 }; const int kSpatialLayerBitrate[3] = { 200, 400, 1000 }; +const int kSpatialLayerBitrateLow[3] = { 50, 100, 400 }; class RcInterfaceTest : public ::libvpx_test::EncoderTest, public ::libvpx_test::CodecTestWith2Params { public: RcInterfaceTest() : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000), - encoder_exit_(false) {} + encoder_exit_(false), frame_drop_thresh_(0), num_drops_(0) {} ~RcInterfaceTest() override = default; protected: void SetUp() override { @@ -74,13 +75,16 @@ return; } int loopfilter_level, qp; encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); - rc_api_->ComputeQP(frame_params_); - ASSERT_EQ(rc_api_->GetQP(), qp); - ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); + if (rc_api_->ComputeQP(frame_params_) == libvpx::FrameDropDecision::kOk) { + ASSERT_EQ(rc_api_->GetQP(), qp); + ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); + } else { + num_drops_++; + } } void FramePktHook(const vpx_codec_cx_pkt_t *pkt) override { rc_api_->PostEncodeUpdate(pkt->data.frame.sz, frame_params_); } @@ -95,10 +99,33 @@ 1280, 720, 30, 1, 0, kNumFrames); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); } + void RunOneLayerDropFramesCBR() { + if (GET_PARAM(2) != VPX_CBR) { + GTEST_SKIP() << "Frame dropping is only for CBR mode."; + } + frame_drop_thresh_ = 30; + SetConfig(GET_PARAM(2)); + // Use lower bitrate, lower max-q, and enable frame dropper. + rc_cfg_.target_bandwidth = 200; + cfg_.rc_target_bitrate = 200; + rc_cfg_.max_quantizer = 50; + cfg_.rc_max_quantizer = 50; + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + frame_params_.spatial_layer_id = 0; + frame_params_.temporal_layer_id = 0; + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Check that some frames were dropped, otherwise test has no value. + ASSERT_GE(num_drops_, 1); + } + void RunOneLayerVBRPeriodicKey() { if (GET_PARAM(2) != VPX_VBR) return; key_interval_ = 100; SetConfig(VPX_VBR); rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); @@ -132,10 +159,11 @@ rc_cfg_.layer_target_bitrate[0] = 1000; rc_cfg_.max_quantizers[0] = 52; rc_cfg_.min_quantizers[0] = 2; rc_cfg_.rc_mode = rc_mode; rc_cfg_.aq_mode = aq_mode_; + rc_cfg_.frame_drop_thresh = frame_drop_thresh_; // Encoder settings for ground truth. cfg_.g_w = 1280; cfg_.g_h = 720; cfg_.rc_undershoot_pct = 50; @@ -150,28 +178,32 @@ cfg_.g_lag_in_frames = 0; cfg_.g_error_resilient = 0; cfg_.rc_target_bitrate = 1000; cfg_.kf_min_dist = key_interval_; cfg_.kf_max_dist = key_interval_; + cfg_.rc_dropframe_thresh = frame_drop_thresh_; } std::unique_ptr rc_api_; libvpx::VP9RateControlRtcConfig rc_cfg_; int aq_mode_; int key_interval_; libvpx::VP9FrameParamsQpRTC frame_params_; bool encoder_exit_; + int frame_drop_thresh_; + int num_drops_; }; class RcInterfaceSvcTest : public ::libvpx_test::EncoderTest, public ::libvpx_test::CodecTestWith2Params { public: RcInterfaceSvcTest() : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000), dynamic_spatial_layers_(0), inter_layer_pred_off_(GET_PARAM(2)), - parallel_spatial_layers_(false) {} + parallel_spatial_layers_(false), frame_drop_thresh_(0), + max_consec_drop_(INT_MAX), num_drops_(0) {} ~RcInterfaceSvcTest() override = default; protected: void SetUp() override { InitializeConfig(); @@ -179,10 +211,11 @@ } void PreEncodeFrameHook(libvpx_test::VideoSource *video, ::libvpx_test::Encoder *encoder) override { if (video->frame() == 0) { + current_superframe_ = 0; encoder->Control(VP8E_SET_CPUUSED, 7); encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); encoder->Control(VP9E_SET_TUNE_CONTENT, 0); encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 900); encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); @@ -190,16 +223,23 @@ encoder->Control(VP9E_SET_SVC_PARAMETERS, &svc_params_); if (inter_layer_pred_off_) { encoder->Control(VP9E_SET_SVC_INTER_LAYER_PRED, INTER_LAYER_PRED_OFF_NONKEY); } + if (frame_drop_thresh_ > 0) { + vpx_svc_frame_drop_t svc_drop_frame; + svc_drop_frame.framedrop_mode = FULL_SUPERFRAME_DROP; + for (int sl = 0; sl < rc_cfg_.ss_number_layers; ++sl) + svc_drop_frame.framedrop_thresh[sl] = frame_drop_thresh_; + svc_drop_frame.max_consec_drop = max_consec_drop_; + encoder->Control(VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame); + } } frame_params_.frame_type = video->frame() % key_interval_ == 0 ? libvpx::RcFrameType::kKeyFrame : libvpx::RcFrameType::kInterFrame; encoder_exit_ = video->frame() == kNumFrames; - current_superframe_ = video->frame(); if (dynamic_spatial_layers_ == 1) { if (video->frame() == 100) { // Go down to 2 spatial layers: set top SL to 0 bitrate. // Update the encoder config. cfg_.rc_target_bitrate -= cfg_.layer_target_bitrate[8]; @@ -255,28 +295,42 @@ ? libvpx::RcFrameType::kKeyFrame : libvpx::RcFrameType::kInterFrame; } void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) override { + if (encoder_exit_) { + return; + } + int superframe_is_dropped = false; ::libvpx_test::CxDataIterator iter = encoder->GetCxData(); for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) sizes_[sl] = 0; std::vector rc_qp; + // For FULL_SUPERFRAME_DROP: the full superframe drop decision is + // determined on the base spatial layer. + SetFrameParamsSvc(0); + if (rc_api_->ComputeQP(frame_params_) == libvpx::FrameDropDecision::kDrop) { + superframe_is_dropped = true; + num_drops_++; + } while (const vpx_codec_cx_pkt_t *pkt = iter.Next()) { + ASSERT_EQ(superframe_is_dropped, false); ParseSuperframeSizes(static_cast(pkt->data.frame.buf), pkt->data.frame.sz); if (!parallel_spatial_layers_ || current_superframe_ == 0) { for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) { if (sizes_[sl] > 0) { SetFrameParamsSvc(sl); - rc_api_->ComputeQP(frame_params_); + // For sl=0 ComputeQP() is already called above (line 310). + if (sl > 0) rc_api_->ComputeQP(frame_params_); rc_api_->PostEncodeUpdate(sizes_[sl], frame_params_); rc_qp.push_back(rc_api_->GetQP()); } } } else { for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) { - if (sizes_[sl] > 0) { + // For sl=0 ComputeQP() is already called above (line 310). + if (sizes_[sl] > 0 && sl > 0) { SetFrameParamsSvc(sl); rc_api_->ComputeQP(frame_params_); } } for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) { @@ -286,18 +340,19 @@ rc_qp.push_back(rc_api_->GetQP()); } } } } - if (!encoder_exit_) { + if (!superframe_is_dropped) { int loopfilter_level; std::vector encoder_qp(VPX_SS_MAX_LAYERS, 0); encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); encoder->Control(VP9E_GET_LAST_QUANTIZER_SVC_LAYERS, encoder_qp.data()); encoder_qp.resize(rc_qp.size()); ASSERT_EQ(rc_qp, encoder_qp); ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); + current_superframe_++; } } // This method needs to be overridden because non-reference frames are // expected to be mismatched frames as the encoder will avoid loopfilter on // these frames. @@ -313,10 +368,25 @@ 1280, 720, 30, 1, 0, kNumFrames); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); } + void RunSvcDropFramesCBR() { + max_consec_drop_ = 10; + frame_drop_thresh_ = 30; + SetRCConfigSvc(3, 3); + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + SetEncoderConfigSvc(3, 3); + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Check that some frames were dropped, otherwise test has no value. + ASSERT_GE(num_drops_, 1); + } + void RunSvcPeriodicKey() { SetRCConfigSvc(3, 3); key_interval_ = 100; rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); SetEncoderConfigSvc(3, 3); @@ -436,16 +506,18 @@ cfg_.rc_max_quantizer = 56; cfg_.g_threads = 1; cfg_.kf_max_dist = 9999; cfg_.rc_overshoot_pct = 50; cfg_.rc_undershoot_pct = 50; + cfg_.rc_dropframe_thresh = frame_drop_thresh_; cfg_.rc_target_bitrate = 0; for (int sl = 0; sl < number_spatial_layers; sl++) { int spatial_bitrate = 0; if (number_spatial_layers <= 3) - spatial_bitrate = kSpatialLayerBitrate[sl]; + spatial_bitrate = frame_drop_thresh_ > 0 ? kSpatialLayerBitrateLow[sl] + : kSpatialLayerBitrate[sl]; for (int tl = 0; tl < number_temporal_layers; tl++) { int layer = sl * number_temporal_layers + tl; if (number_temporal_layers == 3) cfg_.layer_target_bitrate[layer] = kTemporalRateAllocation3Layer[tl] * spatial_bitrate / 100; @@ -476,10 +548,12 @@ rc_cfg_.overshoot_pct = 50; rc_cfg_.max_intra_bitrate_pct = 900; rc_cfg_.framerate = 30.0; rc_cfg_.rc_mode = VPX_CBR; rc_cfg_.aq_mode = aq_mode_; + rc_cfg_.frame_drop_thresh = frame_drop_thresh_; + rc_cfg_.max_consec_drop = max_consec_drop_; if (number_spatial_layers == 3) { rc_cfg_.scaling_factor_num[0] = 1; rc_cfg_.scaling_factor_den[0] = 4; rc_cfg_.scaling_factor_num[1] = 2; @@ -509,11 +583,12 @@ rc_cfg_.target_bandwidth = 0; for (int sl = 0; sl < number_spatial_layers; sl++) { int spatial_bitrate = 0; if (number_spatial_layers <= 3) - spatial_bitrate = kSpatialLayerBitrate[sl]; + spatial_bitrate = frame_drop_thresh_ > 0 ? kSpatialLayerBitrateLow[sl] + : kSpatialLayerBitrate[sl]; for (int tl = 0; tl < number_temporal_layers; tl++) { int layer = sl * number_temporal_layers + tl; if (number_temporal_layers == 3) rc_cfg_.layer_target_bitrate[layer] = kTemporalRateAllocation3Layer[tl] * spatial_bitrate / 100; @@ -546,18 +621,25 @@ int key_interval_; int dynamic_spatial_layers_; bool inter_layer_pred_off_; // ComputeQP() and PostEncodeUpdate() don't need to be sequential for KSVC. bool parallel_spatial_layers_; + int frame_drop_thresh_; + int max_consec_drop_; + int num_drops_; }; TEST_P(RcInterfaceTest, OneLayer) { RunOneLayer(); } +TEST_P(RcInterfaceTest, OneLayerDropFramesCBR) { RunOneLayerDropFramesCBR(); } + TEST_P(RcInterfaceTest, OneLayerVBRPeriodicKey) { RunOneLayerVBRPeriodicKey(); } TEST_P(RcInterfaceSvcTest, Svc) { RunSvc(); } +TEST_P(RcInterfaceSvcTest, SvcDropFramesCBR) { RunSvcDropFramesCBR(); } + TEST_P(RcInterfaceSvcTest, SvcParallelSpatialLayers) { RunSvcParallelSpatialLayers(); } TEST_P(RcInterfaceSvcTest, SvcPeriodicKey) { RunSvcPeriodicKey(); } diff --git a/media/libvpx/libvpx/vp8/common/onyx.h b/media/libvpx/libvpx/vp8/common/onyx.h --- a/media/libvpx/libvpx/vp8/common/onyx.h +++ b/media/libvpx/libvpx/vp8/common/onyx.h @@ -81,11 +81,18 @@ */ int Version; int Width; int Height; struct vpx_rational timebase; - unsigned int target_bandwidth; /* kilobits per second */ + /* In either kilobits per second or bits per second, depending on which + * copy of oxcf this is in. + * - ctx->oxcf.target_bandwidth is in kilobits per second. See + * set_vp8e_config(). + * - ctx->cpi->oxcf.target_bandwidth in is bits per second. See + * vp8_change_config(). + */ + unsigned int target_bandwidth; /* Parameter used for applying denoiser. * For temporal denoiser: noise_sensitivity = 0 means off, * noise_sensitivity = 1 means temporal denoiser on for Y channel only, * noise_sensitivity = 2 means temporal denoiser on for all channels. @@ -212,10 +219,11 @@ vp8e_tuning tuning; /* Temporal scaling parameters */ unsigned int number_of_layers; + /* kilobits per second */ unsigned int target_bitrate[VPX_TS_MAX_PERIODICITY]; unsigned int rate_decimator[VPX_TS_MAX_PERIODICITY]; unsigned int periodicity; unsigned int layer_id[VPX_TS_MAX_PERIODICITY]; @@ -234,11 +242,11 @@ #endif } VP8_CONFIG; void vp8_initialize(); -struct VP8_COMP *vp8_create_compressor(VP8_CONFIG *oxcf); +struct VP8_COMP *vp8_create_compressor(const VP8_CONFIG *oxcf); void vp8_remove_compressor(struct VP8_COMP **comp); void vp8_init_config(struct VP8_COMP *onyx, VP8_CONFIG *oxcf); void vp8_change_config(struct VP8_COMP *cpi, const VP8_CONFIG *oxcf); diff --git a/media/libvpx/libvpx/vp8/encoder/onyx_if.c b/media/libvpx/libvpx/vp8/encoder/onyx_if.c --- a/media/libvpx/libvpx/vp8/encoder/onyx_if.c +++ b/media/libvpx/libvpx/vp8/encoder/onyx_if.c @@ -1288,11 +1288,11 @@ if (cpi->max_gf_interval > cpi->twopass.static_scene_max_gf_interval) { cpi->max_gf_interval = cpi->twopass.static_scene_max_gf_interval; } } -static void init_config(VP8_COMP *cpi, VP8_CONFIG *oxcf) { +static void init_config(VP8_COMP *cpi, const VP8_CONFIG *oxcf) { VP8_COMMON *cm = &cpi->common; cpi->oxcf = *oxcf; cpi->auto_gold = 1; @@ -1414,11 +1414,11 @@ void vp8_change_config(VP8_COMP *cpi, const VP8_CONFIG *oxcf) { VP8_COMMON *cm = &cpi->common; int last_w, last_h; unsigned int prev_number_of_layers; - unsigned int raw_target_rate; + double raw_target_rate; if (!cpi) return; if (!oxcf) return; @@ -1555,14 +1555,14 @@ cpi->oxcf.starting_buffer_level_in_ms = 60000; cpi->oxcf.optimal_buffer_level_in_ms = 60000; cpi->oxcf.maximum_buffer_size_in_ms = 240000; } - raw_target_rate = (unsigned int)((int64_t)cpi->oxcf.Width * cpi->oxcf.Height * - 8 * 3 * cpi->framerate / 1000); + raw_target_rate = ((int64_t)cpi->oxcf.Width * cpi->oxcf.Height * 8 * 3 * + cpi->framerate / 1000.0); if (cpi->oxcf.target_bandwidth > raw_target_rate) - cpi->oxcf.target_bandwidth = raw_target_rate; + cpi->oxcf.target_bandwidth = (unsigned int)raw_target_rate; /* Convert target bandwidth from Kbit/s to Bit/s */ cpi->oxcf.target_bandwidth *= 1000; cpi->oxcf.starting_buffer_level = rescale( (int)cpi->oxcf.starting_buffer_level, cpi->oxcf.target_bandwidth, 1000); @@ -1737,11 +1737,11 @@ mvsadcost[0][-i] = (int)z; mvsadcost[1][-i] = (int)z; } while (++i <= mvfp_max); } -struct VP8_COMP *vp8_create_compressor(VP8_CONFIG *oxcf) { +struct VP8_COMP *vp8_create_compressor(const VP8_CONFIG *oxcf) { int i; VP8_COMP *cpi; VP8_COMMON *cm; @@ -1897,10 +1897,11 @@ cpi->common.refresh_alt_ref_frame = 0; cpi->force_maxqp = 0; cpi->frames_since_last_drop_overshoot = 0; cpi->rt_always_update_correction_factor = 0; + cpi->rt_drop_recode_on_overshoot = 1; cpi->b_calculate_psnr = CONFIG_INTERNAL_STATS; #if CONFIG_INTERNAL_STATS cpi->b_calculate_ssimg = 0; @@ -3181,10 +3182,117 @@ vp8_loop_filter_frame(cm, &cpi->mb.e_mbd, frame_type); } vp8_yv12_extend_frame_borders(cm->frame_to_show); } +// Return 1 if frame is to be dropped. Update frame drop decimation +// counters. +int vp8_check_drop_buffer(VP8_COMP *cpi) { + VP8_COMMON *cm = &cpi->common; + int drop_mark = (int)(cpi->oxcf.drop_frames_water_mark * + cpi->oxcf.optimal_buffer_level / 100); + int drop_mark75 = drop_mark * 2 / 3; + int drop_mark50 = drop_mark / 4; + int drop_mark25 = drop_mark / 8; + if (cpi->drop_frames_allowed) { + /* The reset to decimation 0 is only done here for one pass. + * Once it is set two pass leaves decimation on till the next kf. + */ + if (cpi->buffer_level > drop_mark && cpi->decimation_factor > 0) { + cpi->decimation_factor--; + } + + if (cpi->buffer_level > drop_mark75 && cpi->decimation_factor > 0) { + cpi->decimation_factor = 1; + + } else if (cpi->buffer_level < drop_mark25 && + (cpi->decimation_factor == 2 || cpi->decimation_factor == 3)) { + cpi->decimation_factor = 3; + } else if (cpi->buffer_level < drop_mark50 && + (cpi->decimation_factor == 1 || cpi->decimation_factor == 2)) { + cpi->decimation_factor = 2; + } else if (cpi->buffer_level < drop_mark75 && + (cpi->decimation_factor == 0 || cpi->decimation_factor == 1)) { + cpi->decimation_factor = 1; + } + } + + /* The following decimates the frame rate according to a regular + * pattern (i.e. to 1/2 or 2/3 frame rate) This can be used to help + * prevent buffer under-run in CBR mode. Alternatively it might be + * desirable in some situations to drop frame rate but throw more bits + * at each frame. + * + * Note that dropping a key frame can be problematic if spatial + * resampling is also active + */ + if (cpi->decimation_factor > 0 && cpi->drop_frames_allowed) { + switch (cpi->decimation_factor) { + case 1: + cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 3 / 2; + break; + case 2: + cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 5 / 4; + break; + case 3: + cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 5 / 4; + break; + } + + /* Note that we should not throw out a key frame (especially when + * spatial resampling is enabled). + */ + if (cm->frame_type == KEY_FRAME) { + cpi->decimation_count = cpi->decimation_factor; + } else if (cpi->decimation_count > 0) { + cpi->decimation_count--; + + cpi->bits_off_target += cpi->av_per_frame_bandwidth; + if (cpi->bits_off_target > cpi->oxcf.maximum_buffer_size) { + cpi->bits_off_target = cpi->oxcf.maximum_buffer_size; + } + +#if CONFIG_MULTI_RES_ENCODING + vp8_store_drop_frame_info(cpi); +#endif + + cm->current_video_frame++; + cpi->frames_since_key++; + cpi->ext_refresh_frame_flags_pending = 0; + // We advance the temporal pattern for dropped frames. + cpi->temporal_pattern_counter++; + +#if CONFIG_INTERNAL_STATS + cpi->count++; +#endif + + cpi->buffer_level = cpi->bits_off_target; + + if (cpi->oxcf.number_of_layers > 1) { + unsigned int i; + + /* Propagate bits saved by dropping the frame to higher + * layers + */ + for (i = cpi->current_layer + 1; i < cpi->oxcf.number_of_layers; ++i) { + LAYER_CONTEXT *lc = &cpi->layer_context[i]; + lc->bits_off_target += (int)(lc->target_bandwidth / lc->framerate); + if (lc->bits_off_target > lc->maximum_buffer_size) { + lc->bits_off_target = lc->maximum_buffer_size; + } + lc->buffer_level = lc->bits_off_target; + } + } + return 1; + } else { + cpi->decimation_count = cpi->decimation_factor; + } + } else { + cpi->decimation_count = 0; + } + return 0; +} static void encode_frame_to_data_rate(VP8_COMP *cpi, size_t *size, unsigned char *dest, unsigned char *dest_end, unsigned int *frame_flags) { @@ -3206,16 +3314,10 @@ int bottom_index; int overshoot_seen = 0; int undershoot_seen = 0; #endif - int drop_mark = (int)(cpi->oxcf.drop_frames_water_mark * - cpi->oxcf.optimal_buffer_level / 100); - int drop_mark75 = drop_mark * 2 / 3; - int drop_mark50 = drop_mark / 4; - int drop_mark25 = drop_mark / 8; - /* Clear down mmx registers to allow floating point in what follows */ vpx_clear_system_state(); if (cpi->force_next_frame_intra) { cm->frame_type = KEY_FRAME; /* delayed intra frame */ @@ -3425,106 +3527,12 @@ } #endif update_rd_ref_frame_probs(cpi); - if (cpi->drop_frames_allowed) { - /* The reset to decimation 0 is only done here for one pass. - * Once it is set two pass leaves decimation on till the next kf. - */ - if ((cpi->buffer_level > drop_mark) && (cpi->decimation_factor > 0)) { - cpi->decimation_factor--; - } - - if (cpi->buffer_level > drop_mark75 && cpi->decimation_factor > 0) { - cpi->decimation_factor = 1; - - } else if (cpi->buffer_level < drop_mark25 && - (cpi->decimation_factor == 2 || cpi->decimation_factor == 3)) { - cpi->decimation_factor = 3; - } else if (cpi->buffer_level < drop_mark50 && - (cpi->decimation_factor == 1 || cpi->decimation_factor == 2)) { - cpi->decimation_factor = 2; - } else if (cpi->buffer_level < drop_mark75 && - (cpi->decimation_factor == 0 || cpi->decimation_factor == 1)) { - cpi->decimation_factor = 1; - } - } - - /* The following decimates the frame rate according to a regular - * pattern (i.e. to 1/2 or 2/3 frame rate) This can be used to help - * prevent buffer under-run in CBR mode. Alternatively it might be - * desirable in some situations to drop frame rate but throw more bits - * at each frame. - * - * Note that dropping a key frame can be problematic if spatial - * resampling is also active - */ - if (cpi->decimation_factor > 0 && cpi->drop_frames_allowed) { - switch (cpi->decimation_factor) { - case 1: - cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 3 / 2; - break; - case 2: - cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 5 / 4; - break; - case 3: - cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 5 / 4; - break; - } - - /* Note that we should not throw out a key frame (especially when - * spatial resampling is enabled). - */ - if (cm->frame_type == KEY_FRAME) { - cpi->decimation_count = cpi->decimation_factor; - } else if (cpi->decimation_count > 0) { - cpi->decimation_count--; - - cpi->bits_off_target += cpi->av_per_frame_bandwidth; - if (cpi->bits_off_target > cpi->oxcf.maximum_buffer_size) { - cpi->bits_off_target = cpi->oxcf.maximum_buffer_size; - } - -#if CONFIG_MULTI_RES_ENCODING - vp8_store_drop_frame_info(cpi); -#endif - - cm->current_video_frame++; - cpi->frames_since_key++; - cpi->ext_refresh_frame_flags_pending = 0; - // We advance the temporal pattern for dropped frames. - cpi->temporal_pattern_counter++; - -#if CONFIG_INTERNAL_STATS - cpi->count++; -#endif - - cpi->buffer_level = cpi->bits_off_target; - - if (cpi->oxcf.number_of_layers > 1) { - unsigned int i; - - /* Propagate bits saved by dropping the frame to higher - * layers - */ - for (i = cpi->current_layer + 1; i < cpi->oxcf.number_of_layers; ++i) { - LAYER_CONTEXT *lc = &cpi->layer_context[i]; - lc->bits_off_target += (int)(lc->target_bandwidth / lc->framerate); - if (lc->bits_off_target > lc->maximum_buffer_size) { - lc->bits_off_target = lc->maximum_buffer_size; - } - lc->buffer_level = lc->bits_off_target; - } - } - - return; - } else { - cpi->decimation_count = cpi->decimation_factor; - } - } else { - cpi->decimation_count = 0; + if (vp8_check_drop_buffer(cpi)) { + return; } /* Decide how big to make the frame */ if (!vp8_pick_frame_size(cpi)) { /*TODO: 2 drop_frame and return code could be put together. */ @@ -3928,11 +3936,12 @@ } #else /* transform / motion compensation build reconstruction frame */ vp8_encode_frame(cpi); - if (cpi->pass == 0 && cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) { + if (cpi->pass == 0 && cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER && + cpi->rt_drop_recode_on_overshoot == 1) { if (vp8_drop_encodedframe_overshoot(cpi, Q)) { vpx_clear_system_state(); return; } if (cm->frame_type != KEY_FRAME) diff --git a/media/libvpx/libvpx/vp8/encoder/onyx_int.h b/media/libvpx/libvpx/vp8/encoder/onyx_int.h --- a/media/libvpx/libvpx/vp8/encoder/onyx_int.h +++ b/media/libvpx/libvpx/vp8/encoder/onyx_int.h @@ -213,11 +213,11 @@ }; typedef struct { /* Layer configuration */ double framerate; - int target_bandwidth; + int target_bandwidth; /* bits per second */ /* Layer specific coding parameters */ int64_t starting_buffer_level; int64_t optimal_buffer_level; int64_t maximum_buffer_size; @@ -436,11 +436,11 @@ int gfu_boost; int kf_boost; int last_boost; - int target_bandwidth; + int target_bandwidth; /* bits per second */ struct vpx_codec_pkt_list *output_pkt_list; #if 0 /* Experimental code for lagged and one pass */ ONEPASS_FRAMESTATS one_pass_frame_stats[MAX_LAG_BUFFERS]; @@ -706,10 +706,14 @@ int ext_refresh_frame_flags_pending; // Always update correction factor used for rate control after each frame for // realtime encoding. int rt_always_update_correction_factor; + + // Flag to indicate frame may be dropped due to large expected overshoot, + // and re-encoded on next frame at max_qp. + int rt_drop_recode_on_overshoot; } VP8_COMP; void vp8_initialize_enc(void); void vp8_alloc_compressor_data(VP8_COMP *cpi); @@ -730,10 +734,12 @@ void vp8_tokenize_mb(VP8_COMP *, MACROBLOCK *, TOKENEXTRA **); void vp8_set_speed_features(VP8_COMP *cpi); +int vp8_check_drop_buffer(VP8_COMP *cpi); + #ifdef __cplusplus } // extern "C" #endif #endif // VPX_VP8_ENCODER_ONYX_INT_H_ diff --git a/media/libvpx/libvpx/vp8/encoder/ratectrl.c b/media/libvpx/libvpx/vp8/encoder/ratectrl.c --- a/media/libvpx/libvpx/vp8/encoder/ratectrl.c +++ b/media/libvpx/libvpx/vp8/encoder/ratectrl.c @@ -717,11 +717,12 @@ } else if (percent_low < 0) { percent_low = 0; } /* lower the target bandwidth for this frame. */ - cpi->this_frame_target -= (cpi->this_frame_target * percent_low) / 200; + cpi->this_frame_target -= + (int)(((int64_t)cpi->this_frame_target * percent_low) / 200); /* Are we using allowing control of active_worst_allowed_q * according to buffer level. */ if (cpi->auto_worst_q && cpi->ni_frames > 150) { diff --git a/media/libvpx/libvpx/vp8/vp8_cx_iface.c b/media/libvpx/libvpx/vp8/vp8_cx_iface.c --- a/media/libvpx/libvpx/vp8/vp8_cx_iface.c +++ b/media/libvpx/libvpx/vp8/vp8_cx_iface.c @@ -622,14 +622,15 @@ } static vpx_codec_err_t ctrl_set_rtc_external_ratectrl(vpx_codec_alg_priv_t *ctx, va_list args) { VP8_COMP *cpi = ctx->cpi; - const unsigned int data = CAST(VP8E_SET_GF_CBR_BOOST_PCT, args); + const unsigned int data = CAST(VP8E_SET_RTC_EXTERNAL_RATECTRL, args); if (data) { cpi->cyclic_refresh_mode_enabled = 0; cpi->rt_always_update_correction_factor = 1; + cpi->rt_drop_recode_on_overshoot = 0; } return VPX_CODEC_OK; } static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg, @@ -952,23 +953,14 @@ ctx->timestamp_ratio.num / ctx->timestamp_ratio.den; if (img != NULL) { res = image2yuvconfig(img, &sd); - if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) { - /* from vpx_encoder.h for g_w/g_h: - "Note that the frames passed as input to the encoder must have this - resolution" - */ - ctx->base.err_detail = "Invalid input frame resolution"; - res = VPX_CODEC_INVALID_PARAM; - } else { - if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags, - &sd, dst_time_stamp, dst_end_time_stamp)) { - VP8_COMP *cpi = (VP8_COMP *)ctx->cpi; - res = update_error_state(ctx, &cpi->common.error); - } + if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags, &sd, + dst_time_stamp, dst_end_time_stamp)) { + VP8_COMP *cpi = (VP8_COMP *)ctx->cpi; + res = update_error_state(ctx, &cpi->common.error); } /* reset for next frame */ ctx->next_frame_flag = 0; } diff --git a/media/libvpx/libvpx/vp8/vp8_dx_iface.c b/media/libvpx/libvpx/vp8/vp8_dx_iface.c --- a/media/libvpx/libvpx/vp8/vp8_dx_iface.c +++ b/media/libvpx/libvpx/vp8/vp8_dx_iface.c @@ -18,10 +18,11 @@ #include "vpx/vp8dx.h" #include "vpx/internal/vpx_codec_internal.h" #include "vpx_version.h" #include "common/alloccommon.h" #include "common/common.h" +#include "common/onyxc_int.h" #include "common/onyxd.h" #include "decoder/onyxd_int.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_mem/vpx_mem.h" #include "vpx_ports/system_state.h" @@ -247,18 +248,18 @@ } if (ctx->fragments.enabled && !(data == NULL && data_sz == 0)) { /* Store a pointer to this fragment and return. We haven't * received the complete frame yet, so we will wait with decoding. */ - ctx->fragments.ptrs[ctx->fragments.count] = data; - ctx->fragments.sizes[ctx->fragments.count] = data_sz; - ctx->fragments.count++; - if (ctx->fragments.count > (1 << EIGHT_PARTITION) + 1) { + if (ctx->fragments.count >= MAX_PARTITIONS) { ctx->fragments.count = 0; *res = VPX_CODEC_INVALID_PARAM; return -1; } + ctx->fragments.ptrs[ctx->fragments.count] = data; + ctx->fragments.sizes[ctx->fragments.count] = data_sz; + ctx->fragments.count++; return 0; } if (!ctx->fragments.enabled && (data == NULL && data_sz == 0)) { return 0; diff --git a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h --- a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h +++ b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h @@ -31,10 +31,15 @@ struct VP8FrameParamsQpRTC { RcFrameType frame_type; int temporal_layer_id; }; +enum class FrameDropDecision { + kOk, // Frame is encoded. + kDrop, // Frame is dropped. +}; + class VP8RateControlRTC { public: static std::unique_ptr Create( const VP8RateControlRtcConfig &cfg); ~VP8RateControlRTC(); @@ -43,12 +48,14 @@ // GetQP() needs to be called after ComputeQP() to get the latest QP int GetQP() const; // GetLoopfilterLevel() needs to be called after ComputeQP() since loopfilter // level is calculated from frame qp. int GetLoopfilterLevel() const; - // int GetLoopfilterLevel() const; - void ComputeQP(const VP8FrameParamsQpRTC &frame_params); + // ComputeQP computes the QP if the frame is not dropped (kOk return), + // otherwise it returns kDrop and subsequent GetQP and PostEncodeUpdate + // are not to be called. + FrameDropDecision ComputeQP(const VP8FrameParamsQpRTC &frame_params); // Feedback to rate control with the size of current encoded frame void PostEncodeUpdate(uint64_t encoded_frame_size); private: VP8RateControlRTC() {} diff --git a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc --- a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc +++ b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc @@ -131,10 +131,12 @@ oxcf->maximum_buffer_size = rc_cfg.buf_sz; oxcf->number_of_layers = rc_cfg.ts_number_layers; cpi_->buffered_mode = oxcf->optimal_buffer_level > 0; oxcf->under_shoot_pct = rc_cfg.undershoot_pct; oxcf->over_shoot_pct = rc_cfg.overshoot_pct; + oxcf->drop_frames_water_mark = rc_cfg.frame_drop_thresh; + if (oxcf->drop_frames_water_mark > 0) cpi_->drop_frames_allowed = 1; cpi_->oxcf.rc_max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct; cpi_->framerate = rc_cfg.framerate; for (int i = 0; i < KEY_FRAME_CONTEXT; ++i) { cpi_->prior_key_frame_distance[i] = static_cast(cpi_->output_framerate); @@ -206,11 +208,12 @@ vp8_new_framerate(cpi_, cpi_->framerate); vpx_clear_system_state(); return true; } -void VP8RateControlRTC::ComputeQP(const VP8FrameParamsQpRTC &frame_params) { +FrameDropDecision VP8RateControlRTC::ComputeQP( + const VP8FrameParamsQpRTC &frame_params) { VP8_COMMON *const cm = &cpi_->common; vpx_clear_system_state(); if (cpi_->oxcf.number_of_layers > 1) { cpi_->temporal_layer_id = frame_params.temporal_layer_id; const int layer = frame_params.temporal_layer_id; @@ -224,11 +227,24 @@ cm->refresh_alt_ref_frame = (cm->frame_type == KEY_FRAME) ? 1 : 0; if (cm->frame_type == KEY_FRAME && cpi_->common.current_video_frame > 0) { cpi_->common.frame_flags |= FRAMEFLAGS_KEY; } - vp8_pick_frame_size(cpi_); + cpi_->per_frame_bandwidth = static_cast( + round(cpi_->oxcf.target_bandwidth / cpi_->output_framerate)); + if (vp8_check_drop_buffer(cpi_)) { + if (cpi_->oxcf.number_of_layers > 1) vp8_save_layer_context(cpi_); + return FrameDropDecision::kDrop; + } + + if (!vp8_pick_frame_size(cpi_)) { + cm->current_video_frame++; + cpi_->frames_since_key++; + cpi_->ext_refresh_frame_flags_pending = 0; + if (cpi_->oxcf.number_of_layers > 1) vp8_save_layer_context(cpi_); + return FrameDropDecision::kDrop; + } if (cpi_->buffer_level >= cpi_->oxcf.optimal_buffer_level && cpi_->buffered_mode) { /* Max adjustment is 1/4 */ int Adjustment = cpi_->active_worst_quality / 4; @@ -288,10 +304,11 @@ } q_ = vp8_regulate_q(cpi_, cpi_->this_frame_target); vp8_set_quantizer(cpi_, q_); vpx_clear_system_state(); + return FrameDropDecision::kOk; } int VP8RateControlRTC::GetQP() const { return q_; } int VP8RateControlRTC::GetLoopfilterLevel() const { diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c b/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c @@ -1299,10 +1299,17 @@ // the reference (base layer frame) is key frame (i.e., is_key_frame == 1). int is_key_frame = (frame_is_intra_only(cm) || (is_one_pass_svc(cpi) && cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame)); + + if (!is_key_frame) { + if (cm->frame_refs[LAST_FRAME - 1].sf.x_scale_fp == REF_INVALID_SCALE || + cm->frame_refs[LAST_FRAME - 1].sf.y_scale_fp == REF_INVALID_SCALE) + is_key_frame = 1; + } + // Always use 4x4 partition for key frame. const int use_4x4_partition = frame_is_intra_only(cm); const int low_res = (cm->width <= 352 && cm->height <= 288); int variance4x4downsample[16]; int segment_id; @@ -3050,18 +3057,16 @@ BLOCK_SIZE bs, min_size, max_size; min_size = BLOCK_64X64; max_size = BLOCK_4X4; - if (prev_mi) { - for (idy = 0; idy < mi_height; ++idy) { - for (idx = 0; idx < mi_width; ++idx) { - mi = prev_mi[idy * cm->mi_stride + idx]; - bs = mi ? mi->sb_type : bsize; - min_size = VPXMIN(min_size, bs); - max_size = VPXMAX(max_size, bs); - } + for (idy = 0; idy < mi_height; ++idy) { + for (idx = 0; idx < mi_width; ++idx) { + mi = prev_mi[idy * cm->mi_stride + idx]; + bs = mi ? mi->sb_type : bsize; + min_size = VPXMIN(min_size, bs); + max_size = VPXMAX(max_size, bs); } } if (xd->left_mi) { for (idy = 0; idy < mi_height; ++idy) { @@ -3203,11 +3208,11 @@ left_par = 2; else if (context_size == bsize) left_par = 1; } - if (prev_mi) { + if (prev_mi[0]) { context_size = prev_mi[0]->sb_type; if (context_size < bsize) last_par = 2; else if (context_size == bsize) last_par = 1; @@ -6137,10 +6142,19 @@ if (tpl_frame->is_valid) cpi->rd.r0 = (double)intra_cost_base / mc_dep_cost_base; } + for (MV_REFERENCE_FRAME ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; + ++ref_frame) { + if (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) { + if (cm->frame_refs[ref_frame - 1].sf.x_scale_fp == REF_INVALID_SCALE || + cm->frame_refs[ref_frame - 1].sf.y_scale_fp == REF_INVALID_SCALE) + cpi->ref_frame_flags &= ~ref_frame_to_flag(ref_frame); + } + } + // Frame segmentation if (cpi->oxcf.aq_mode == PERCEPTUAL_AQ) build_kmeans_segmentation(cpi); { struct vpx_usec_timer emr_timer; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c @@ -5515,30 +5515,11 @@ VP9_COMMON *const cm = &cpi->common; const VP9EncoderConfig *const oxcf = &cpi->oxcf; struct segmentation *const seg = &cm->seg; TX_SIZE t; - // SVC: skip encoding of enhancement layer if the layer target bandwidth = 0. - // No need to set svc.skip_enhancement_layer if whole superframe will be - // dropped. - if (cpi->use_svc && cpi->svc.spatial_layer_id > 0 && - cpi->oxcf.target_bandwidth == 0 && - !(cpi->svc.framedrop_mode != LAYER_DROP && - (cpi->svc.framedrop_mode != CONSTRAINED_FROM_ABOVE_DROP || - cpi->svc - .force_drop_constrained_from_above[cpi->svc.number_spatial_layers - - 1]) && - cpi->svc.drop_spatial_layer[0])) { - cpi->svc.skip_enhancement_layer = 1; - vp9_rc_postencode_update_drop_frame(cpi); - cpi->ext_refresh_frame_flags_pending = 0; - cpi->last_frame_dropped = 1; - cpi->svc.last_layer_dropped[cpi->svc.spatial_layer_id] = 1; - cpi->svc.drop_spatial_layer[cpi->svc.spatial_layer_id] = 1; - vp9_inc_frame_in_layer(cpi); - return; - } + if (vp9_svc_check_skip_enhancement_layer(cpi)) return; set_ext_overrides(cpi); vpx_clear_system_state(); #ifdef ENABLE_KF_DENOISE diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.h b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.h @@ -348,10 +348,12 @@ void vp9_estimate_qp_gop(struct VP9_COMP *cpi); void vp9_compute_frame_low_motion(struct VP9_COMP *const cpi); +void vp9_update_buffer_level_svc_preencode(struct VP9_COMP *cpi); + #ifdef __cplusplus } // extern "C" #endif #endif // VPX_VP9_ENCODER_VP9_RATECTRL_H_ diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c @@ -258,11 +258,11 @@ // Update the buffer level before encoding with the per-frame-bandwidth // for SVC. The current and all upper temporal layers are updated, needed // for the layered rate control which involves cumulative buffer levels for // the temporal layers. Allow for using the timestamp(pts) delta for the // framerate when the set_ref_frame_config is used. -static void update_buffer_level_svc_preencode(VP9_COMP *cpi) { +void vp9_update_buffer_level_svc_preencode(VP9_COMP *cpi) { SVC *const svc = &cpi->svc; int i; // Set this to 1 to use timestamp delta for "framerate" under // ref_frame_config usage. int use_timestamp = 1; @@ -2443,11 +2443,11 @@ // should be done here, before the frame qp is selected. if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) vp9_cyclic_refresh_update_parameters(cpi); vp9_rc_set_frame_target(cpi, target); - if (cm->show_frame) update_buffer_level_svc_preencode(cpi); + if (cm->show_frame) vp9_update_buffer_level_svc_preencode(cpi); if (cpi->oxcf.resize_mode == RESIZE_DYNAMIC && svc->single_layer_svc == 1 && svc->spatial_layer_id == svc->first_spatial_layer_to_encode && svc->temporal_layer_id == 0) { LAYER_CONTEXT *lc = NULL; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c b/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c @@ -3604,11 +3604,11 @@ cpi->rc.frames_since_golden >= 3) if (x->pred_mv_sad[GOLDEN_FRAME] > (x->pred_mv_sad[LAST_FRAME] << 1)) mode_skip_mask[GOLDEN_FRAME] |= INTER_ALL; } - if (bsize > sf->max_intra_bsize) { + if (bsize > sf->max_intra_bsize && cpi->ref_frame_flags != 0) { ref_frame_skip_mask[0] |= (1 << INTRA_FRAME); ref_frame_skip_mask[1] |= (1 << INTRA_FRAME); } mode_skip_mask[INTRA_FRAME] |= diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.h b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.h @@ -279,10 +279,12 @@ void vp9_svc_update_ref_frame(struct VP9_COMP *const cpi); void vp9_svc_adjust_frame_rate(struct VP9_COMP *const cpi); void vp9_svc_adjust_avg_frame_qindex(struct VP9_COMP *const cpi); + +int vp9_svc_check_skip_enhancement_layer(struct VP9_COMP *const cpi); #ifdef __cplusplus } // extern "C" #endif #endif // VPX_VP9_ENCODER_VP9_SVC_LAYERCONTEXT_H_ diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c @@ -221,15 +221,15 @@ lc->spatial_layer_target_bandwidth = spatial_layer_target; if (target_bandwidth != 0) { bitrate_alloc = (float)lc->target_bandwidth / target_bandwidth; } lrc->starting_buffer_level = - (int64_t)(rc->starting_buffer_level * bitrate_alloc); + (int64_t)(rc->starting_buffer_level * bitrate_alloc + 0.5); lrc->optimal_buffer_level = - (int64_t)(rc->optimal_buffer_level * bitrate_alloc); + (int64_t)(rc->optimal_buffer_level * bitrate_alloc + 0.5); lrc->maximum_buffer_size = - (int64_t)(rc->maximum_buffer_size * bitrate_alloc); + (int64_t)(rc->maximum_buffer_size * bitrate_alloc + 0.5); lrc->bits_off_target = VPXMIN(lrc->bits_off_target, lrc->maximum_buffer_size); lrc->buffer_level = VPXMIN(lrc->buffer_level, lrc->maximum_buffer_size); lc->framerate = cpi->framerate / oxcf->ts_rate_decimator[tl]; lrc->avg_frame_bandwidth = @@ -1348,5 +1348,29 @@ RATE_CONTROL *lrc = &lc->rc; lrc->avg_frame_qindex[INTER_FRAME] = rc->avg_frame_qindex[INTER_FRAME]; } } } + +// SVC: skip encoding of enhancement layer if the layer target bandwidth = 0. +// No need to set svc.skip_enhancement_layer if whole superframe will be +// dropped. +int vp9_svc_check_skip_enhancement_layer(VP9_COMP *const cpi) { + if (cpi->use_svc && cpi->svc.spatial_layer_id > 0 && + cpi->oxcf.target_bandwidth == 0 && + !(cpi->svc.framedrop_mode != LAYER_DROP && + (cpi->svc.framedrop_mode != CONSTRAINED_FROM_ABOVE_DROP || + cpi->svc + .force_drop_constrained_from_above[cpi->svc.number_spatial_layers - + 1]) && + cpi->svc.drop_spatial_layer[0])) { + cpi->svc.skip_enhancement_layer = 1; + vp9_rc_postencode_update_drop_frame(cpi); + cpi->ext_refresh_frame_flags_pending = 0; + cpi->last_frame_dropped = 1; + cpi->svc.last_layer_dropped[cpi->svc.spatial_layer_id] = 1; + cpi->svc.drop_spatial_layer[cpi->svc.spatial_layer_id] = 1; + vp9_inc_frame_in_layer(cpi); + return 1; + } + return 0; +} diff --git a/media/libvpx/libvpx/vp9/ratectrl_rtc.h b/media/libvpx/libvpx/vp9/ratectrl_rtc.h --- a/media/libvpx/libvpx/vp9/ratectrl_rtc.h +++ b/media/libvpx/libvpx/vp9/ratectrl_rtc.h @@ -36,18 +36,21 @@ vp9_zero(ts_rate_decimator); scaling_factor_num[0] = 1; scaling_factor_den[0] = 1; max_quantizers[0] = max_quantizer; min_quantizers[0] = min_quantizer; + max_consec_drop = INT_MAX; } // Number of spatial layers int ss_number_layers; int max_quantizers[VPX_MAX_LAYERS]; int min_quantizers[VPX_MAX_LAYERS]; int scaling_factor_num[VPX_SS_MAX_LAYERS]; int scaling_factor_den[VPX_SS_MAX_LAYERS]; + // This is only for SVC for now. + int max_consec_drop; }; struct VP9FrameParamsQpRTC { RcFrameType frame_type; int spatial_layer_id; @@ -59,10 +62,15 @@ size_t segmentation_map_size; const int *delta_q; size_t delta_q_size; }; +enum class FrameDropDecision { + kOk, // Frame is encoded. + kDrop, // Frame is dropped. +}; + // This interface allows using VP9 real-time rate control without initializing // the encoder. To use this interface, you need to link with libvpxrc.a. // // #include "vp9/ratectrl_rtc.h" // VP9RateControlRtcConfig cfg; @@ -90,11 +98,15 @@ bool UpdateRateControl(const VP9RateControlRtcConfig &rc_cfg); // GetQP() needs to be called after ComputeQP() to get the latest QP int GetQP() const; int GetLoopfilterLevel() const; bool GetSegmentationData(VP9SegmentationData *segmentation_data) const; - void ComputeQP(const VP9FrameParamsQpRTC &frame_params); + // ComputeQP computes the QP if the frame is not dropped (kOk return), + // otherwise it returns kDrop and subsequent GetQP and PostEncodeUpdate + // are not to be called (vp9_rc_postencode_update_drop_frame is already + // called via ComputeQP if drop is decided). + FrameDropDecision ComputeQP(const VP9FrameParamsQpRTC &frame_params); // Feedback to rate control with the size of current encoded frame void PostEncodeUpdate(uint64_t encoded_frame_size, const VP9FrameParamsQpRTC &frame_params); private: diff --git a/media/libvpx/libvpx/vp9/ratectrl_rtc.cc b/media/libvpx/libvpx/vp9/ratectrl_rtc.cc --- a/media/libvpx/libvpx/vp9/ratectrl_rtc.cc +++ b/media/libvpx/libvpx/vp9/ratectrl_rtc.cc @@ -128,10 +128,11 @@ oxcf->starting_buffer_level_ms = rc_cfg.buf_initial_sz; oxcf->optimal_buffer_level_ms = rc_cfg.buf_optimal_sz; oxcf->maximum_buffer_size_ms = rc_cfg.buf_sz; oxcf->under_shoot_pct = rc_cfg.undershoot_pct; oxcf->over_shoot_pct = rc_cfg.overshoot_pct; + oxcf->drop_frames_water_mark = rc_cfg.frame_drop_thresh; oxcf->ss_number_layers = rc_cfg.ss_number_layers; oxcf->ts_number_layers = rc_cfg.ts_number_layers; oxcf->temporal_layering_mode = (VP9E_TEMPORAL_LAYERING_MODE)( (rc_cfg.ts_number_layers > 1) ? rc_cfg.ts_number_layers : 0); @@ -170,21 +171,31 @@ } vp9_set_rc_buffer_sizes(cpi_); vp9_new_framerate(cpi_, cpi_->framerate); if (cpi_->svc.number_temporal_layers > 1 || cpi_->svc.number_spatial_layers > 1) { - if (cm->current_video_frame == 0) vp9_init_layer_context(cpi_); + if (cm->current_video_frame == 0) { + vp9_init_layer_context(cpi_); + // svc->framedrop_mode is not currently exposed, so only allow for + // full superframe drop for now. + cpi_->svc.framedrop_mode = FULL_SUPERFRAME_DROP; + } vp9_update_layer_context_change_config(cpi_, (int)cpi_->oxcf.target_bandwidth); + cpi_->svc.max_consec_drop = rc_cfg.max_consec_drop; } vp9_check_reset_rc_flag(cpi_); cpi_->common.error.setjmp = 0; return true; } -void VP9RateControlRTC::ComputeQP(const VP9FrameParamsQpRTC &frame_params) { +// Compute the QP for the frame. If the frame is dropped this function +// returns kDrop, and no QP is computed. If the frame is encoded (not dropped) +// the QP is computed and kOk is returned. +FrameDropDecision VP9RateControlRTC::ComputeQP( + const VP9FrameParamsQpRTC &frame_params) { VP9_COMMON *const cm = &cpi_->common; int width, height; cpi_->svc.spatial_layer_id = frame_params.spatial_layer_id; cpi_->svc.temporal_layer_id = frame_params.temporal_layer_id; if (cpi_->svc.number_spatial_layers > 1) { @@ -232,18 +243,55 @@ } else { vp9_update_temporal_layer_framerate(cpi_); vp9_restore_layer_context(cpi_); vp9_rc_get_svc_params(cpi_); } + if (cpi_->svc.spatial_layer_id == 0) vp9_zero(cpi_->svc.drop_spatial_layer); + // SVC: check for skip encoding of enhancement layer if the + // layer target bandwidth = 0. + if (vp9_svc_check_skip_enhancement_layer(cpi_)) + return FrameDropDecision::kDrop; + // Check for dropping this frame based on buffer level. + // Never drop on key frame, or if base layer is key for svc, + if (!frame_is_intra_only(cm) && + (!cpi_->use_svc || + !cpi_->svc.layer_context[cpi_->svc.temporal_layer_id].is_key_frame)) { + if (vp9_rc_drop_frame(cpi_)) { + // For FULL_SUPERFRAME_DROP mode (the only mode considered here): + // if the superframe drop is decided we need to save the layer context for + // all spatial layers, and call update_buffer_level and postencode_drop + // for all spatial layers. + if (cpi_->svc.number_spatial_layers > 1 || + cpi_->svc.number_temporal_layers > 1) { + vp9_save_layer_context(cpi_); + for (int sl = 1; sl < cpi_->svc.number_spatial_layers; sl++) { + cpi_->svc.spatial_layer_id = sl; + vp9_restore_layer_context(cpi_); + vp9_update_buffer_level_svc_preencode(cpi_); + vp9_rc_postencode_update_drop_frame(cpi_); + vp9_save_layer_context(cpi_); + } + } + return FrameDropDecision::kDrop; + } + } + // Compute the QP for the frame. int bottom_index, top_index; cpi_->common.base_qindex = vp9_rc_pick_q_and_bounds(cpi_, &bottom_index, &top_index); if (cpi_->oxcf.aq_mode == CYCLIC_REFRESH_AQ) vp9_cyclic_refresh_setup(cpi_); if (cpi_->svc.number_spatial_layers > 1 || cpi_->svc.number_temporal_layers > 1) vp9_save_layer_context(cpi_); + + cpi_->last_frame_dropped = 0; + cpi_->svc.last_layer_dropped[cpi_->svc.spatial_layer_id] = 0; + if (cpi_->svc.spatial_layer_id == cpi_->svc.number_spatial_layers - 1) + cpi_->svc.num_encoded_top_layer++; + + return FrameDropDecision::kOk; } int VP9RateControlRTC::GetQP() const { return cpi_->common.base_qindex; } int VP9RateControlRTC::GetLoopfilterLevel() const { diff --git a/media/libvpx/libvpx/vp9/vp9_cx_iface.c b/media/libvpx/libvpx/vp9/vp9_cx_iface.c --- a/media/libvpx/libvpx/vp9/vp9_cx_iface.c +++ b/media/libvpx/libvpx/vp9/vp9_cx_iface.c @@ -1379,26 +1379,17 @@ if (img != NULL) { const int64_t dst_end_time_stamp = timebase_units_to_ticks(timestamp_ratio, pts + duration); res = image2yuvconfig(img, &sd); - if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) { - /* from vpx_encoder.h for g_w/g_h: - "Note that the frames passed as input to the encoder must have this - resolution" - */ - ctx->base.err_detail = "Invalid input frame resolution"; - res = VPX_CODEC_INVALID_PARAM; - } else { - // Store the original flags in to the frame buffer. Will extract the - // key frame flag when we actually encode this frame. - if (vp9_receive_raw_frame(cpi, flags | ctx->next_frame_flags, &sd, + // Store the original flags in to the frame buffer. Will extract the + // key frame flag when we actually encode this frame. + if (vp9_receive_raw_frame(cpi, flags | ctx->next_frame_flags, &sd, dst_time_stamp, dst_end_time_stamp)) { - res = update_error_state(ctx, &cpi->common.error); - } - ctx->next_frame_flags = 0; + res = update_error_state(ctx, &cpi->common.error); } + ctx->next_frame_flags = 0; } cx_data = ctx->cx_data; cx_data_sz = ctx->cx_data_sz; diff --git a/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h b/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h --- a/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h +++ b/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h @@ -35,10 +35,11 @@ ts_number_layers = 1; rc_mode = VPX_CBR; aq_mode = 0; layer_target_bitrate[0] = static_cast(target_bandwidth); ts_rate_decimator[0] = 1; + frame_drop_thresh = 0; } int width; int height; // 0-63 @@ -58,8 +59,9 @@ int layer_target_bitrate[VPX_MAX_LAYERS]; int ts_rate_decimator[VPX_TS_MAX_LAYERS]; // vbr, cbr enum vpx_rc_mode rc_mode; int aq_mode; + int frame_drop_thresh; }; } // namespace libvpx #endif // VPX_VPX_INTERNAL_VPX_RATECTRL_RTC_H_ diff --git a/media/libvpx/libvpx/vpx/vp8cx.h b/media/libvpx/libvpx/vpx/vp8cx.h --- a/media/libvpx/libvpx/vpx/vp8cx.h +++ b/media/libvpx/libvpx/vpx/vp8cx.h @@ -164,10 +164,11 @@ * of motion estimation methods. Values greater than 0 will increase encoder * speed at the expense of quality. * * \note Valid range for VP8: -16..16 * \note Valid range for VP9: -9..9 + * \note A negative value (-n) is treated as its absolute value (n) in VP9. * * Supported in codecs: VP8, VP9 */ VP8E_SET_CPUUSED = 13, diff --git a/media/libvpx/libvpx/vpx_dsp/x86/subtract_sse2.asm b/media/libvpx/libvpx/vpx_dsp/x86/subtract_sse2.asm --- a/media/libvpx/libvpx/vpx_dsp/x86/subtract_sse2.asm +++ b/media/libvpx/libvpx/vpx_dsp/x86/subtract_sse2.asm @@ -122,6 +122,7 @@ lea diffq, [diffq+diff_strideq*4] lea srcq, [srcq+src_strideq*2] lea predq, [predq+pred_str*2] sub rowsd, 2 jg .loop_4 + emms RET diff --git a/media/libvpx/libvpx/vpx_ports/aarch64_cpudetect.c b/media/libvpx/libvpx/vpx_ports/aarch64_cpudetect.c --- a/media/libvpx/libvpx/vpx_ports/aarch64_cpudetect.c +++ b/media/libvpx/libvpx/vpx_ports/aarch64_cpudetect.c @@ -102,11 +102,13 @@ #define VPX_AARCH64_HWCAP2_I8MM (1 << 13) static int arm_get_cpu_caps(void) { int flags = 0; unsigned long hwcap = getauxval(AT_HWCAP); +#if HAVE_NEON_I8MM unsigned long hwcap2 = getauxval(AT_HWCAP2); +#endif // HAVE_NEON_I8MM #if HAVE_NEON flags |= HAS_NEON; // Neon is mandatory in Armv8.0-A. #endif // HAVE_NEON #if HAVE_NEON_DOTPROD if (hwcap & VPX_AARCH64_HWCAP_ASIMDDP) { diff --git a/media/libvpx/libvpx/vpxenc.c b/media/libvpx/libvpx/vpxenc.c --- a/media/libvpx/libvpx/vpxenc.c +++ b/media/libvpx/libvpx/vpxenc.c @@ -881,11 +881,11 @@ /* Allows removal of the application version from the EBML tags */ stream->webm_ctx.debug = global->debug; /* Default lag_in_frames is 0 in realtime mode CBR mode*/ if (global->deadline == VPX_DL_REALTIME && - stream->config.cfg.rc_end_usage == 1) + stream->config.cfg.rc_end_usage == VPX_CBR) stream->config.cfg.g_lag_in_frames = 0; } /* Output files must be specified for each stream */ stream->config.out_fn = NULL; diff --git a/media/libvpx/moz.yaml b/media/libvpx/moz.yaml --- a/media/libvpx/moz.yaml +++ b/media/libvpx/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://chromium.googlesource.com/webm/libvpx # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: b759032a0ed2b57ea3412f6820eda377a2dad480 (Fri Oct 27 17:02:46 2023). + release: d7358ed53a6b344f6f374b3a5e8fe3d207bcc720 (Mon Nov 27 17:35:10 2023). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: b759032a0ed2b57ea3412f6820eda377a2dad480 + revision: d7358ed53a6b344f6f374b3a5e8fe3d207bcc720 # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text