/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdio.h>
#include <deque>
#include <map>
#include "testing/gtest/include/gtest/gtest.h"
#include "webrtc/base/thread_annotations.h"
#include "webrtc/call.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
#include "webrtc/system_wrappers/interface/clock.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/event_wrapper.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
#include "webrtc/system_wrappers/interface/sleep.h"
#include "webrtc/test/call_test.h"
#include "webrtc/test/direct_transport.h"
#include "webrtc/test/encoder_settings.h"
#include "webrtc/test/fake_encoder.h"
#include "webrtc/test/frame_generator_capturer.h"
#include "webrtc/test/statistics.h"
#include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/typedefs.h"
namespace webrtc {
static const int kFullStackTestDurationSecs = 60;
struct FullStackTestParams {
const char* test_label;
struct {
const char* name;
size_t width, height;
int fps;
} clip;
int min_bitrate_bps;
int target_bitrate_bps;
int max_bitrate_bps;
double avg_psnr_threshold;
double avg_ssim_threshold;
FakeNetworkPipe::Config link;
};
class FullStackTest : public test::CallTest {
protected:
void RunTest(const FullStackTestParams& params);
};
class VideoAnalyzer : public PacketReceiver,
public newapi::Transport,
public VideoRenderer,
public VideoSendStreamInput {
public:
VideoAnalyzer(VideoSendStreamInput* input,
Transport* transport,
const char* test_label,
double avg_psnr_threshold,
double avg_ssim_threshold,
int duration_frames)
: input_(input),
transport_(transport),
receiver_(NULL),
test_label_(test_label),
frames_left_(duration_frames),
dropped_frames_(0),
last_render_time_(0),
rtp_timestamp_delta_(0),
crit_(CriticalSectionWrapper::CreateCriticalSection()),
first_send_frame_(NULL),
avg_psnr_threshold_(avg_psnr_threshold),
avg_ssim_threshold_(avg_ssim_threshold),
comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
comparison_thread_(ThreadWrapper::CreateThread(&FrameComparisonThread,
this)),
done_(EventWrapper::Create()) {
unsigned int id;
EXPECT_TRUE(comparison_thread_->Start(id));
}
~VideoAnalyzer() {
EXPECT_TRUE(comparison_thread_->Stop());
while (!frames_.empty()) {
delete frames_.back();
frames_.pop_back();
}
while (!frame_pool_.empty()) {
delete frame_pool_.back();
frame_pool_.pop_back();
}
}
virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
virtual DeliveryStatus DeliverPacket(const uint8_t* packet,
size_t length) OVERRIDE {
scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
RTPHeader header;
parser->Parse(packet, length, &header);
{
CriticalSectionScoped lock(crit_.get());
recv_times_[header.timestamp - rtp_timestamp_delta_] =
Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
}
return receiver_->DeliverPacket(packet, length);
}
virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE {
I420VideoFrame* copy = NULL;
{
CriticalSectionScoped lock(crit_.get());
if (frame_pool_.size() > 0) {
copy = frame_pool_.front();
frame_pool_.pop_front();
}
}
if (copy == NULL)
copy = new I420VideoFrame();
copy->CopyFrame(*video_frame);
copy->set_timestamp(copy->render_time_ms() * 90);
{
CriticalSectionScoped lock(crit_.get());
if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
first_send_frame_ = copy;
frames_.push_back(copy);
}
input_->SwapFrame(video_frame);
}
virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE {
scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
RTPHeader header;
parser->Parse(packet, length, &header);
{
CriticalSectionScoped lock(crit_.get());
if (rtp_timestamp_delta_ == 0) {
rtp_timestamp_delta_ =
header.timestamp - first_send_frame_->timestamp();
first_send_frame_ = NULL;
}
send_times_[header.timestamp - rtp_timestamp_delta_] =
Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
}
return transport_->SendRtp(packet, length);
}
virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE {
return transport_->SendRtcp(packet, length);
}
virtual void RenderFrame(const I420VideoFrame& video_frame,
int time_to_render_ms) OVERRIDE {
int64_t render_time_ms =
Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
CriticalSectionScoped lock(crit_.get());
while (frames_.front()->timestamp() < send_timestamp) {
AddFrameComparison(
frames_.front(), &last_rendered_frame_, true, render_time_ms);
frame_pool_.push_back(frames_.front());
frames_.pop_front();
}
I420VideoFrame* reference_frame = frames_.front();
frames_.pop_front();
assert(reference_frame != NULL);
EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
assert(reference_frame->timestamp() == send_timestamp);
AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
frame_pool_.push_back(reference_frame);
last_rendered_frame_.CopyFrame(video_frame);
}
void Wait() {
EXPECT_EQ(kEventSignaled, done_->Wait(FullStackTest::kLongTimeoutMs));
}
VideoSendStreamInput* input_;
Transport* transport_;
PacketReceiver* receiver_;
private:
struct FrameComparison {
FrameComparison(const I420VideoFrame* reference,
const I420VideoFrame* render,
bool dropped,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms)
: dropped(dropped),
send_time_ms(send_time_ms),
recv_time_ms(recv_time_ms),
render_time_ms(render_time_ms) {
this->reference.CopyFrame(*reference);
this->render.CopyFrame(*render);
}
FrameComparison(const FrameComparison& compare)
: dropped(compare.dropped),
send_time_ms(compare.send_time_ms),
recv_time_ms(compare.recv_time_ms),
render_time_ms(compare.render_time_ms) {
this->reference.CopyFrame(compare.reference);
this->render.CopyFrame(compare.render);
}
~FrameComparison() {}
I420VideoFrame reference;
I420VideoFrame render;
bool dropped;
int64_t send_time_ms;
int64_t recv_time_ms;
int64_t render_time_ms;
};
void AddFrameComparison(const I420VideoFrame* reference,
const I420VideoFrame* render,
bool dropped,
int64_t render_time_ms)
EXCLUSIVE_LOCKS_REQUIRED(crit_) {
int64_t send_time_ms = send_times_[reference->timestamp()];
send_times_.erase(reference->timestamp());
int64_t recv_time_ms = recv_times_[reference->timestamp()];
recv_times_.erase(reference->timestamp());
CriticalSectionScoped crit(comparison_lock_.get());
comparisons_.push_back(FrameComparison(reference,
render,
dropped,
send_time_ms,
recv_time_ms,
render_time_ms));
}
static bool FrameComparisonThread(void* obj) {
return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
}
bool CompareFrames() {
assert(frames_left_ > 0);
I420VideoFrame reference;
I420VideoFrame render;
bool dropped;
int64_t send_time_ms;
int64_t recv_time_ms;
int64_t render_time_ms;
SleepMs(10);
while (true) {
{
CriticalSectionScoped crit(comparison_lock_.get());
if (comparisons_.empty())
return true;
reference.SwapFrame(&comparisons_.front().reference);
render.SwapFrame(&comparisons_.front().render);
dropped = comparisons_.front().dropped;
send_time_ms = comparisons_.front().send_time_ms;
recv_time_ms = comparisons_.front().recv_time_ms;
render_time_ms = comparisons_.front().render_time_ms;
comparisons_.pop_front();
}
PerformFrameComparison(&reference,
&render,
dropped,
send_time_ms,
recv_time_ms,
render_time_ms);
if (--frames_left_ == 0) {
PrintResult("psnr", psnr_, " dB");
PrintResult("ssim", ssim_, "");
PrintResult("sender_time", sender_time_, " ms");
printf(
"RESULT dropped_frames: %s = %d\n", test_label_, dropped_frames_);
PrintResult("receiver_time", receiver_time_, " ms");
PrintResult("total_delay_incl_network", end_to_end_, " ms");
PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
done_->Set();
return false;
}
}
}
void PerformFrameComparison(const I420VideoFrame* reference,
const I420VideoFrame* render,
bool dropped,
int64_t send_time_ms,
int64_t recv_time_ms,
int64_t render_time_ms) {
psnr_.AddSample(I420PSNR(reference, render));
ssim_.AddSample(I420SSIM(reference, render));
if (dropped) {
++dropped_frames_;
return;
}
if (last_render_time_ != 0)
rendered_delta_.AddSample(render_time_ms - last_render_time_);
last_render_time_ = render_time_ms;
int64_t input_time_ms = reference->render_time_ms();
sender_time_.AddSample(send_time_ms - input_time_ms);
receiver_time_.AddSample(render_time_ms - recv_time_ms);
end_to_end_.AddSample(render_time_ms - input_time_ms);
}
void PrintResult(const char* result_type,
test::Statistics stats,
const char* unit) {
printf("RESULT %s: %s = {%f, %f}%s\n",
result_type,
test_label_,
stats.Mean(),
stats.StandardDeviation(),
unit);
}
const char* const test_label_;
test::Statistics sender_time_;
test::Statistics receiver_time_;
test::Statistics psnr_;
test::Statistics ssim_;
test::Statistics end_to_end_;
test::Statistics rendered_delta_;
int frames_left_;
int dropped_frames_;
int64_t last_render_time_;
uint32_t rtp_timestamp_delta_;
const scoped_ptr<CriticalSectionWrapper> crit_;
std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
double avg_psnr_threshold_ GUARDED_BY(crit_);
double avg_ssim_threshold_ GUARDED_BY(crit_);
const scoped_ptr<CriticalSectionWrapper> comparison_lock_;
const scoped_ptr<ThreadWrapper> comparison_thread_;
std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
const scoped_ptr<EventWrapper> done_;
};
void FullStackTest::RunTest(const FullStackTestParams& params) {
test::DirectTransport send_transport(params.link);
test::DirectTransport recv_transport(params.link);
VideoAnalyzer analyzer(NULL,
&send_transport,
params.test_label,
params.avg_psnr_threshold,
params.avg_ssim_threshold,
kFullStackTestDurationSecs * params.clip.fps);
CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
analyzer.SetReceiver(receiver_call_->Receiver());
send_transport.SetReceiver(&analyzer);
recv_transport.SetReceiver(sender_call_->Receiver());
CreateSendConfig(1);
scoped_ptr<VideoEncoder> encoder(
VideoEncoder::Create(VideoEncoder::kVp8));
send_config_.encoder_settings.encoder = encoder.get();
send_config_.encoder_settings.payload_name = "VP8";
send_config_.encoder_settings.payload_type = 124;
VideoStream* stream = &encoder_config_.streams[0];
stream->width = params.clip.width;
stream->height = params.clip.height;
stream->min_bitrate_bps = params.min_bitrate_bps;
stream->target_bitrate_bps = params.target_bitrate_bps;
stream->max_bitrate_bps = params.max_bitrate_bps;
stream->max_framerate = params.clip.fps;
CreateMatchingReceiveConfigs();
receive_configs_[0].renderer = &analyzer;
CreateStreams();
analyzer.input_ = send_stream_->Input();
frame_generator_capturer_.reset(
test::FrameGeneratorCapturer::CreateFromYuvFile(
&analyzer,
test::ResourcePath(params.clip.name, "yuv").c_str(),
params.clip.width,
params.clip.height,
params.clip.fps,
Clock::GetRealTimeClock()));
ASSERT_TRUE(frame_generator_capturer_.get() != NULL)
<< "Could not create capturer for " << params.clip.name
<< ".yuv. Is this resource file present?";
Start();
analyzer.Wait();
send_transport.StopSending();
recv_transport.StopSending();
Stop();
DestroyStreams();
}
TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
{"paris_qcif", 176, 144, 30},
300000,
300000,
300000,
36.0,
0.96
};
RunTest(paris_qcif);
}
TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
// TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
{"foreman_cif", 352, 288, 30},
700000,
700000,
700000,
0.0,
0.0
};
RunTest(foreman_cif);
}
TEST_F(FullStackTest, ForemanCif500kbps) {
FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
{"foreman_cif", 352, 288, 30},
30000,
500000,
2000000,
0.0,
0.0
};
foreman_cif.link.queue_length_packets = 0;
foreman_cif.link.queue_delay_ms = 0;
foreman_cif.link.link_capacity_kbps = 500;
RunTest(foreman_cif);
}
TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue",
{"foreman_cif", 352, 288, 30},
30000,
500000,
2000000,
0.0,
0.0
};
foreman_cif.link.queue_length_packets = 32;
foreman_cif.link.queue_delay_ms = 0;
foreman_cif.link.link_capacity_kbps = 500;
RunTest(foreman_cif);
}
TEST_F(FullStackTest, ForemanCif500kbps100ms) {
FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms",
{"foreman_cif", 352, 288, 30},
30000,
500000,
2000000,
0.0,
0.0
};
foreman_cif.link.queue_length_packets = 0;
foreman_cif.link.queue_delay_ms = 100;
foreman_cif.link.link_capacity_kbps = 500;
RunTest(foreman_cif);
}
TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue",
{"foreman_cif", 352, 288, 30},
30000,
500000,
2000000,
0.0,
0.0
};
foreman_cif.link.queue_length_packets = 32;
foreman_cif.link.queue_delay_ms = 100;
foreman_cif.link.link_capacity_kbps = 500;
RunTest(foreman_cif);
}
TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue",
{"foreman_cif", 352, 288, 30},
30000,
2000000,
2000000,
0.0,
0.0
};
foreman_cif.link.queue_length_packets = 32;
foreman_cif.link.queue_delay_ms = 100;
foreman_cif.link.link_capacity_kbps = 1000;
RunTest(foreman_cif);
}
} // namespace webrtc