webrtc-分辨率控制之量化参数
1.资源管理逻辑
在Webrtc的视频会议过程中，rtc底层会根据网络情况和设备运行情况动态的调整视频的分辨率，帧率，码率等，以控制发送的数据量，避免在网络不佳的情况下因丢包导致的卡顿或花屏。那它是如何实现这一功能的呢？在webrtc中它将网络带宽和cpu抽象成了一个资源，通过监控这些资源的使用情况，并及时的调整资源使用策略。如何调整呢？通过降低或提高视频质量（分辨率，帧率，码率）就可以降低和增加资源的使用率。
在这一过程中，需要介绍以下相关对象：
BandwidthQualityScalerResource:带宽资源的抽象，监听带宽的使用情况
EncodeUsageResource:编码器资源(或cpu)的抽象，监听编码器的使用情况
QualityScalerResource：视频质量的抽象，监听编码后帧数据的qp值，根据qp值来控制是否应该调整分辨率
VideoStreamAdapter:资源使用者，当资源使用状况发生变化时需要调整使用策略
ResourceAdaptationProcessor:资源与资源使用者之间的桥梁，当资源使用状况发生变化时先通知给它
VideoStreamEncoderResourceManager：管理上面提到的这些抽象资源，在构造时，会创建上述Resource对象
DegradationPreferences:降级策略，有DISABLED，MAINTAIN_FRAMERATE，MAINTAIN_RESOLUTION和BALANCED等四种选项，可以在java上层配置。
这些对象会在创建VideoStreamEncoder时创建，这里就不去跟踪源码了，下面列出其调用栈。
WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream
WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec
WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream
webrtc::VideoSendStream* Call::CreateVideoSendStream
VideoSendStream::VideoSendStream
std::unique_ptr<VideoStreamEncoder> CreateVideoStreamEncoder
VideoStreamEncoder::VideoStreamEncoder
2.量化参数
量化参数（QP）是用于控制编码器对视频帧中每个像素值的量化程度。QP 值越高，量化程度越高，视频质量越低，带宽消耗越小。QP 值越低，量化程度越低，视频质量越高，带宽消耗越大。H.264 中，QP 值的取值范围为 0 到 51，其中 0 表示最精细的量化，51 表示最粗糙的量化。
下面先看一下 QualityScalerResource是如何工作的。这里我们以使用硬编码的方式进行代码跟踪。
1.从java层获取到qp配置
在native创建好Encoder后会更新编码器相关数据，其中有一步就是拿到Java层Encoder对于QP值的配置。
VideoEncoderWrapper::VideoEncoderWrapper(JNIEnv* jni,
const JavaRef<jobject>& j_encoder)
: encoder_(jni, j_encoder), int_array_class_(GetClass(jni, "[I")) {
initialized_ = false;
num_resets_ = 0;
// Fetch and update encoder info.
UpdateEncoderInfo(jni);
}
void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv* jni) {
encoder_info_.supports_native_handle = true;
encoder_info_.implementation_name = JavaToStdString(
jni, Java_VideoEncoder_getImplementationName(jni, encoder_));
encoder_info_.is_hardware_accelerated =
Java_VideoEncoder_isHardwareEncoder(jni, encoder_);
encoder_info_.scaling_settings = GetScalingSettingsInternal(jni);
encoder_info_.resolution_bitrate_limits = JavaToNativeResolutionBitrateLimits(
jni, Java_VideoEncoder_getResolutionBitrateLimits(jni, encoder_));
EncoderInfo info = GetEncoderInfoInternal(jni);
encoder_info_.requested_resolution_alignment =
info.requested_resolution_alignment;
encoder_info_.apply_alignment_to_all_simulcast_layers =
info.apply_alignment_to_all_simulcast_layers;
}
VideoEncoderWrapper::ScalingSettings
VideoEncoderWrapper::GetScalingSettingsInternal(JNIEnv* jni) const {
// 调用java层的getScalingSettings
ScopedJavaLocalRef<jobject> j_scaling_settings =
Java_VideoEncoder_getScalingSettings(jni, encoder_);
bool isOn =
Java_VideoEncoderWrapper_getScalingSettingsOn(jni, j_scaling_settings);
if (!isOn)
return ScalingSettings::kOff;
absl::optional<int> low = JavaToNativeOptionalInt(
jni,
Java_VideoEncoderWrapper_getScalingSettingsLow(jni, j_scaling_settings));
absl::optional<int> high = JavaToNativeOptionalInt(
jni,
Java_VideoEncoderWrapper_getScalingSettingsHigh(jni, j_scaling_settings));
if (low && high)
return ScalingSettings(*low, *high);
}
// HardwareVideoEncoder.java
public ScalingSettings getScalingSettings() {
encodeThreadChecker.checkIsOnValidThread();
if (automaticResizeOn) {
if (codecType == VideoCodecMimeType.VP8) {
final int kLowVp8QpThreshold = 29;
final int kHighVp8QpThreshold = 95;
return new ScalingSettings(kLowVp8QpThreshold, kHighVp8QpThreshold);
} else if (codecType == VideoCodecMimeType.H264) {
final int kLowH264QpThreshold = 24;
final int kHighH264QpThreshold = 37;
return new ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold);
}
}
return ScalingSettings.OFF;
}
2.初始化QualityScalerResource
void VideoStreamEncoder::SetSource(
rtc::VideoSourceInterface<VideoFrame>* source,
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.SetSource(source);
input_state_provider_.OnHasInputChanged(source);
// This may trigger reconfiguring the QualityScaler on the encoder queue.
encoder_queue_.PostTask([this, degradation_preference] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
degradation_preference_manager_->SetDegradationPreference(
degradation_preference);
stream_resource_manager_.SetDegradationPreferences(degradation_preference);
if (encoder_) {
// 初始化
stream_resource_manager_.ConfigureQualityScaler(
encoder_->GetEncoderInfo());
stream_resource_manager_.ConfigureBandwidthQualityScaler(
encoder_->GetEncoderInfo());
}
});
}
void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) {
RTC_DCHECK_RUN_ON(encoder_queue_);
// 获取配置
const auto scaling_settings = encoder_info.scaling_settings;
// 是否运行分辨率缩放
const bool quality_scaling_allowed =
IsResolutionScalingEnabled(degradation_preference_) &&
(scaling_settings.thresholds.has_value() ||
(encoder_settings_.has_value() &&
encoder_settings_->encoder_config().is_quality_scaling_allowed)) &&
encoder_info.is_qp_trusted.value_or(true);
if (quality_scaling_allowed) {
if (!quality_scaler_resource_->is_started()) {
// Quality scaler has not already been configured.
// Use experimental thresholds if available.
absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
if (quality_scaling_experiment_enabled_) {
experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
GetVideoCodecTypeOrGeneric(encoder_settings_));
}
// 更新配置并开始检测
UpdateQualityScalerSettings(experimental_thresholds.has_value()
? experimental_thresholds
: scaling_settings.thresholds);
}
} else {
UpdateQualityScalerSettings(absl::nullopt);
}
// Set the qp-thresholds to the balanced settings if balanced mode.
if (degradation_preference_ == DegradationPreference::BALANCED &&
quality_scaler_resource_->is_started()) {
absl::optional<VideoEncoder::QpThresholds> thresholds =
balanced_settings_.GetQpThresholds(
GetVideoCodecTypeOrGeneric(encoder_settings_),
LastFrameSizeOrDefault());
if (thresholds) {
quality_scaler_resource_->SetQpThresholds(*thresholds);
}
}
UpdateStatsAdaptationSettings();
}
3.启动检测
void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
RTC_DCHECK_RUN_ON(encoder_queue_);
if (qp_thresholds.has_value()) {
if (quality_scaler_resource_->is_started()) {
quality_scaler_resource_->SetQpThresholds(qp_thresholds.value());
} else {
// 执行检测任务
quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value());
AddResource(quality_scaler_resource_, VideoAdaptationReason::kQuality);
}
} else if (quality_scaler_resource_->is_started()) {
// 结束检测
quality_scaler_resource_->StopCheckForOveruse();
RemoveResource(quality_scaler_resource_);
}
initial_frame_dropper_->OnQualityScalerSettingsUpdated();
}
void QualityScalerResource::StartCheckForOveruse(
VideoEncoder::QpThresholds qp_thresholds) {
RTC_DCHECK_RUN_ON(encoder_queue());
RTC_DCHECK(!is_started());
quality_scaler_ =
std::make_unique<QualityScaler>(this, std::move(qp_thresholds));
}
QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds,
int64_t default_sampling_period_ms)
: handler_(handler),
thresholds_(thresholds),
sampling_period_ms_(QualityScalerSettings::ParseFromFieldTrials()
.SamplingPeriodMs()
.value_or(default_sampling_period_ms)),
fast_rampup_(true),
// Arbitrarily choose size based on 30 fps for 5 seconds.
average_qp_(QualityScalerSettings::ParseFromFieldTrials()
.AverageQpWindow()
.value_or(5 * 30)),
framedrop_percent_media_opt_(5 * 30),
framedrop_percent_all_(5 * 30),
experiment_enabled_(QualityScalingExperiment::Enabled()),
min_frames_needed_(
QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
kMinFramesNeededToScale)),
initial_scale_factor_(QualityScalerSettings::ParseFromFieldTrials()
.InitialScaleFactor()
.value_or(kSamplePeriodScaleFactor)),
scale_factor_(
QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) {
RTC_DCHECK_RUN_ON(&task_checker_);
if (experiment_enabled_) {
config_ = QualityScalingExperiment::GetConfig();
// qp_smoother_high_ 高频，会随着qp值的变化，快速响应变化；qp_smoother_low_ 低频，随着qp值的变化，响应变化较慢
qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
}
RTC_DCHECK(handler_ != nullptr);
StartNextCheckQpTask();
RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
<< ", high: " << thresholds_.high;
}
void VideoStreamEncoderResourceManager::AddResource(
rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason) {
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(resource);
bool inserted;
std::tie(std::ignore, inserted) = resources_.emplace(resource, reason);
adaptation_processor_->AddResource(resource);
}
void ResourceAdaptationProcessor::AddResource(
rtc::scoped_refptr<Resource> resource) {
{
MutexLock crit(&resources_lock_);
resources_.push_back(resource);
}
// resource_listener_delegate_的作用是切换线程，本身还是当前ResourceAdaptationProcessor对象
resource->SetResourceListener(resource_listener_delegate_.get());
}
4.当有一帧视频被编码
当一帧视频被解码后，会计算出这一帧的qp值。QualityScalerResource收到这个值后会这个样本其添加到average_qp_ ，qp_smoother_high_，qp_smoother_high_中用于计算平均值。
void QualityScalerResource::OnEncodeCompleted(const EncodedImage& encoded_image,
int64_t time_sent_in_us) {
RTC_DCHECK_RUN_ON(encoder_queue());
if (quality_scaler_ && encoded_image.qp_ >= 0) {
quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us);
}
}
void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
RTC_DCHECK_RUN_ON(&task_checker_);
// 帧计数和丢帧率计算
framedrop_percent_media_opt_.AddSample(0);
framedrop_percent_all_.AddSample(0);
// average_qp_ 平滑窗口计算出的平均值
average_qp_.AddSample(qp);
if (qp_smoother_high_)
qp_smoother_high_->Add(qp, time_sent_us);
if (qp_smoother_high_)
qp_smoother_low_->Add(qp, time_sent_us);
}
5.执行检测
在下面的CheckQp方法中已经添加了比较详细的注释，其中qp_smoother_high_高频，会随着qp值的变化，快速响应变化，因此在qp值升高的情况下，其得到的值会较快速升高，更少的样本就能达到阈值。而对于qp_smoother_low_则是相反的。这里的返回值决定了资源的使用情况，从而对资源使用进行调整，可以看出对于资源使用的提升是更谨慎的。当然，这只是我个人的理解，如果有错误的地方欢迎指正。
void QualityScaler::StartNextCheckQpTask() {
CheckQpTask::Result previous_task_result;
if (pending_qp_task_) {
previous_task_result = pending_qp_task_->result();
}
pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
pending_qp_task_->StartDelayedTask();
}
void StartDelayedTask() {
RTC_DCHECK_EQ(state_, State::kNotStarted);
state_ = State::kCheckingQp;
TaskQueueBase::Current()->PostDelayedTask(
[this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
if (!this_weak_ptr) {
// The task has been cancelled through destruction.
return;
}
// 执行检测
switch (quality_scaler_->CheckQp()) {
case QualityScaler::CheckQpResult::kInsufficientSamples: {
result_.observed_enough_frames = false;
// After this line, `this` may be deleted.
break;
}
case QualityScaler::CheckQpResult::kNormalQp: {
result_.observed_enough_frames = true;
break;
}
case QualityScaler::CheckQpResult::kHighQp: {
result_.observed_enough_frames = true;
result_.qp_usage_reported = true;
quality_scaler_->fast_rampup_ = false;
// 通知handler_
quality_scaler_->handler_->OnReportQpUsageHigh();
// 清空所有的样本
quality_scaler_->ClearSamples();
break;
}
case QualityScaler::CheckQpResult::kLowQp: {
result_.observed_enough_frames = true;
result_.qp_usage_reported = true;
quality_scaler_->handler_->OnReportQpUsageLow();
quality_scaler_->ClearSamples();
break;
}
}
state_ = State::kCompleted;
// Starting the next task deletes the pending task. After this line,
// `this` has been deleted.
quality_scaler_->StartNextCheckQpTask();
},
TimeDelta::Millis(GetCheckingQpDelayMs()));
}
QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
// 帧的数量是否足够
const size_t frames = config_.use_all_drop_reasons
? framedrop_percent_all_.Size()
: framedrop_percent_media_opt_.Size();
if (frames < min_frames_needed_) {
return CheckQpResult::kInsufficientSamples;
}
// 如果丢帧率比较高，那么返回kHighQp
const absl::optional<int> drop_rate =
config_.use_all_drop_reasons
? framedrop_percent_all_.GetAverageRoundedDown()
: framedrop_percent_media_opt_.GetAverageRoundedDown();
if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
return CheckQpResult::kHighQp;
}
// Check if we should scale up or down based on QP.
const absl::optional<int> avg_qp_high =
qp_smoother_high_ ? qp_smoother_high_->GetAvg()
: average_qp_.GetAverageRoundedDown();
const absl::optional<int> avg_qp_low =
qp_smoother_low_ ? qp_smoother_low_->GetAvg()
: average_qp_.GetAverageRoundedDown();
if (avg_qp_high && avg_qp_low) {
// 如果最近的平均QP值大于高阈值，返回kHighQp
if (*avg_qp_high > thresholds_.high) {
return CheckQpResult::kHighQp;
}
// 如果上面的条件不满足，再看一下较长时间内的平均QP值小于低阈值，返回kLowQp
if (*avg_qp_low <= thresholds_.low) {
// QP has been low. We want to try a higher resolution.
return CheckQpResult::kLowQp;
}
}
return CheckQpResult::kNormalQp;
}
6.通知检测结果
最终会调用到上一篇（分辨率切换逻辑）文章的开头，在那里进行相关资源使用策略的变更。
以上是个人见解，如果错误或不同见解，欢迎指正和交流。
