// Copyright 2013 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "media/cast/video_sender/video_sender.h" #include <list> #include "base/bind.h" #include "base/logging.h" #include "base/message_loop/message_loop.h" #include "crypto/encryptor.h" #include "crypto/symmetric_key.h" #include "media/cast/cast_defines.h" #include "media/cast/net/pacing/paced_sender.h" #include "media/cast/video_sender/video_encoder.h" namespace media { namespace cast { const int64 kMinSchedulingDelayMs = 1; class LocalRtcpVideoSenderFeedback : public RtcpSenderFeedback { public: explicit LocalRtcpVideoSenderFeedback(VideoSender* video_sender) : video_sender_(video_sender) { } virtual void OnReceivedCastFeedback( const RtcpCastMessage& cast_feedback) OVERRIDE { video_sender_->OnReceivedCastFeedback(cast_feedback); } private: VideoSender* video_sender_; }; class LocalRtpVideoSenderStatistics : public RtpSenderStatistics { public: explicit LocalRtpVideoSenderStatistics(RtpSender* rtp_sender) : rtp_sender_(rtp_sender) { } virtual void GetStatistics(const base::TimeTicks& now, RtcpSenderInfo* sender_info) OVERRIDE { rtp_sender_->RtpStatistics(now, sender_info); } private: RtpSender* rtp_sender_; }; VideoSender::VideoSender( scoped_refptr<CastEnvironment> cast_environment, const VideoSenderConfig& video_config, VideoEncoderController* const video_encoder_controller, PacedPacketSender* const paced_packet_sender) : rtp_max_delay_( base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)), max_frame_rate_(video_config.max_frame_rate), cast_environment_(cast_environment), rtcp_feedback_(new LocalRtcpVideoSenderFeedback(this)), rtp_sender_(new RtpSender(cast_environment, NULL, &video_config, paced_packet_sender)), last_acked_frame_id_(-1), last_sent_frame_id_(-1), duplicate_ack_(0), last_skip_count_(0), congestion_control_(cast_environment->Clock(), video_config.congestion_control_back_off, video_config.max_bitrate, video_config.min_bitrate, video_config.start_bitrate), initialized_(false), weak_factory_(this) { max_unacked_frames_ = static_cast<uint8>(video_config.rtp_max_delay_ms * video_config.max_frame_rate / 1000) + 1; VLOG(1) << "max_unacked_frames " << static_cast<int>(max_unacked_frames_); DCHECK_GT(max_unacked_frames_, 0) << "Invalid argument"; rtp_video_sender_statistics_.reset( new LocalRtpVideoSenderStatistics(rtp_sender_.get())); if (video_config.use_external_encoder) { DCHECK(video_encoder_controller) << "Invalid argument"; video_encoder_controller_ = video_encoder_controller; } else { video_encoder_.reset(new VideoEncoder(cast_environment, video_config, max_unacked_frames_)); video_encoder_controller_ = video_encoder_.get(); } if (video_config.aes_iv_mask.size() == kAesKeySize && video_config.aes_key.size() == kAesKeySize) { iv_mask_ = video_config.aes_iv_mask; crypto::SymmetricKey* key = crypto::SymmetricKey::Import( crypto::SymmetricKey::AES, video_config.aes_key); encryptor_.reset(new crypto::Encryptor()); encryptor_->Init(key, crypto::Encryptor::CTR, std::string()); } else if (video_config.aes_iv_mask.size() != 0 || video_config.aes_key.size() != 0) { DCHECK(false) << "Invalid crypto configuration"; } rtcp_.reset(new Rtcp( cast_environment_, rtcp_feedback_.get(), paced_packet_sender, rtp_video_sender_statistics_.get(), NULL, video_config.rtcp_mode, base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), video_config.sender_ssrc, video_config.incoming_feedback_ssrc, video_config.rtcp_c_name)); } VideoSender::~VideoSender() {} void VideoSender::InitializeTimers() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (!initialized_) { initialized_ = true; ScheduleNextRtcpReport(); ScheduleNextResendCheck(); ScheduleNextSkippedFramesCheck(); } } void VideoSender::InsertRawVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& capture_time) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); DCHECK(video_encoder_.get()) << "Invalid state"; cast_environment_->Logging()->InsertFrameEvent(kVideoFrameReceived, GetVideoRtpTimestamp(capture_time), kFrameIdUnknown); if (!video_encoder_->EncodeVideoFrame(video_frame, capture_time, base::Bind(&VideoSender::SendEncodedVideoFrameMainThread, weak_factory_.GetWeakPtr()))) { } } void VideoSender::InsertCodedVideoFrame(const EncodedVideoFrame* encoded_frame, const base::TimeTicks& capture_time, const base::Closure callback) { DCHECK(!video_encoder_.get()) << "Invalid state"; DCHECK(encoded_frame) << "Invalid argument"; DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); SendEncodedVideoFrame(encoded_frame, capture_time); cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback); } void VideoSender::SendEncodedVideoFrameMainThread( scoped_ptr<EncodedVideoFrame> video_frame, const base::TimeTicks& capture_time) { SendEncodedVideoFrame(video_frame.get(), capture_time); } bool VideoSender::EncryptVideoFrame(const EncodedVideoFrame& video_frame, EncodedVideoFrame* encrypted_frame) { DCHECK(encryptor_) << "Invalid state"; if (!encryptor_->SetCounter(GetAesNonce(video_frame.frame_id, iv_mask_))) { NOTREACHED() << "Failed to set counter"; return false; } if (!encryptor_->Encrypt(video_frame.data, &encrypted_frame->data)) { NOTREACHED() << "Encrypt error"; return false; } encrypted_frame->codec = video_frame.codec; encrypted_frame->key_frame = video_frame.key_frame; encrypted_frame->frame_id = video_frame.frame_id; encrypted_frame->last_referenced_frame_id = video_frame.last_referenced_frame_id; return true; } void VideoSender::SendEncodedVideoFrame(const EncodedVideoFrame* encoded_frame, const base::TimeTicks& capture_time) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); last_send_time_ = cast_environment_->Clock()->NowTicks(); if (encryptor_) { EncodedVideoFrame encrypted_video_frame; if (!EncryptVideoFrame(*encoded_frame, &encrypted_video_frame)) { // Logging already done. return; } rtp_sender_->IncomingEncodedVideoFrame(&encrypted_video_frame, capture_time); } else { rtp_sender_->IncomingEncodedVideoFrame(encoded_frame, capture_time); } if (encoded_frame->key_frame) { VLOG(1) << "Send encoded key frame; frame_id:" << static_cast<int>(encoded_frame->frame_id); } last_sent_frame_id_ = static_cast<int>(encoded_frame->frame_id); UpdateFramesInFlight(); InitializeTimers(); } void VideoSender::IncomingRtcpPacket(const uint8* packet, size_t length, const base::Closure callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); rtcp_->IncomingRtcpPacket(packet, length); cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback); } void VideoSender::ScheduleNextRtcpReport() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() - cast_environment_->Clock()->NowTicks(); time_to_next = std::max(time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoSender::SendRtcpReport, weak_factory_.GetWeakPtr()), time_to_next); } void VideoSender::SendRtcpReport() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); RtcpSenderLogMessage sender_log_message; const FrameRawMap& frame_raw_map = cast_environment_->Logging()->GetFrameRawData(); FrameRawMap::const_iterator it = frame_raw_map.begin(); while (it != frame_raw_map.end()) { RtcpSenderFrameLogMessage frame_message; frame_message.rtp_timestamp = it->first; frame_message.frame_status = kRtcpSenderFrameStatusUnknown; if (it->second.type.empty()) { ++it; continue; } CastLoggingEvent last_event = it->second.type.back(); switch (last_event) { case kVideoFrameCaptured: frame_message.frame_status = kRtcpSenderFrameStatusDroppedByFlowControl; break; case kVideoFrameSentToEncoder: frame_message.frame_status = kRtcpSenderFrameStatusDroppedByEncoder; break; case kVideoFrameEncoded: frame_message.frame_status = kRtcpSenderFrameStatusSentToNetwork; break; default: ++it; continue; } ++it; if (it == frame_raw_map.end()) { // Last message on our map; only send if it is kVideoFrameEncoded. if (last_event != kVideoFrameEncoded) { // For other events we will wait for it to finish and report the result // in the next report. break; } } sender_log_message.push_back(frame_message); } rtcp_->SendRtcpFromRtpSender(&sender_log_message); if (!sender_log_message.empty()) { VLOG(1) << "Failed to send all log messages"; } // TODO(pwestin): When we start pulling out the logging by other means we need // to synchronize this. cast_environment_->Logging()->Reset(); ScheduleNextRtcpReport(); } void VideoSender::ScheduleNextResendCheck() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeDelta time_to_next; if (last_send_time_.is_null()) { time_to_next = rtp_max_delay_; } else { time_to_next = last_send_time_ - cast_environment_->Clock()->NowTicks() + rtp_max_delay_; } time_to_next = std::max(time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoSender::ResendCheck, weak_factory_.GetWeakPtr()), time_to_next); } void VideoSender::ResendCheck() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (!last_send_time_.is_null() && last_sent_frame_id_ != -1) { base::TimeDelta time_since_last_send = cast_environment_->Clock()->NowTicks() - last_send_time_; if (time_since_last_send > rtp_max_delay_) { if (last_acked_frame_id_ == -1) { // We have not received any ack, send a key frame. video_encoder_controller_->GenerateKeyFrame(); last_acked_frame_id_ = -1; last_sent_frame_id_ = -1; UpdateFramesInFlight(); } else { DCHECK_LE(0, last_acked_frame_id_); uint32 frame_id = static_cast<uint32>(last_acked_frame_id_ + 1); VLOG(1) << "ACK timeout resend frame:" << static_cast<int>(frame_id); ResendFrame(frame_id); } } } ScheduleNextResendCheck(); } void VideoSender::ScheduleNextSkippedFramesCheck() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeDelta time_to_next; if (last_checked_skip_count_time_.is_null()) { time_to_next = base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs); } else { time_to_next = last_checked_skip_count_time_ - cast_environment_->Clock()->NowTicks() + base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs); } time_to_next = std::max(time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoSender::SkippedFramesCheck, weak_factory_.GetWeakPtr()), time_to_next); } void VideoSender::SkippedFramesCheck() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); int skip_count = video_encoder_controller_->NumberOfSkippedFrames(); if (skip_count - last_skip_count_ > kSkippedFramesThreshold * max_frame_rate_) { // TODO(pwestin): Propagate this up to the application. } last_skip_count_ = skip_count; last_checked_skip_count_time_ = cast_environment_->Clock()->NowTicks(); ScheduleNextSkippedFramesCheck(); } void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeDelta rtt; base::TimeDelta avg_rtt; base::TimeDelta min_rtt; base::TimeDelta max_rtt; if (rtcp_->Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) { cast_environment_->Logging()->InsertGenericEvent(kRttMs, rtt.InMilliseconds()); // Don't use a RTT lower than our average. rtt = std::max(rtt, avg_rtt); } else { // We have no measured value use default. rtt = base::TimeDelta::FromMilliseconds(kStartRttMs); } if (cast_feedback.missing_frames_and_packets_.empty()) { // No lost packets. int resend_frame = -1; if (last_sent_frame_id_ == -1) return; video_encoder_controller_->LatestFrameIdToReference( cast_feedback.ack_frame_id_); if (static_cast<uint32>(last_acked_frame_id_ + 1) == cast_feedback.ack_frame_id_) { uint32 new_bitrate = 0; if (congestion_control_.OnAck(rtt, &new_bitrate)) { video_encoder_controller_->SetBitRate(new_bitrate); } } if (static_cast<uint32>(last_acked_frame_id_) == cast_feedback.ack_frame_id_ // We only count duplicate ACKs when we have sent newer frames. && IsNewerFrameId(last_sent_frame_id_, last_acked_frame_id_)) { duplicate_ack_++; } else { duplicate_ack_ = 0; } if (duplicate_ack_ >= 2 && duplicate_ack_ % 3 == 2) { // Resend last ACK + 1 frame. resend_frame = static_cast<uint32>(last_acked_frame_id_ + 1); } if (resend_frame != -1) { DCHECK_LE(0, resend_frame); VLOG(1) << "Received duplicate ACK for frame:" << static_cast<int>(resend_frame); ResendFrame(static_cast<uint32>(resend_frame)); } } else { rtp_sender_->ResendPackets(cast_feedback.missing_frames_and_packets_); last_send_time_ = cast_environment_->Clock()->NowTicks(); uint32 new_bitrate = 0; if (congestion_control_.OnNack(rtt, &new_bitrate)) { video_encoder_controller_->SetBitRate(new_bitrate); } } ReceivedAck(cast_feedback.ack_frame_id_); } void VideoSender::ReceivedAck(uint32 acked_frame_id) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); last_acked_frame_id_ = static_cast<int>(acked_frame_id); cast_environment_->Logging()->InsertGenericEvent(kAckReceived, acked_frame_id); VLOG(1) << "ReceivedAck:" << static_cast<int>(acked_frame_id); last_acked_frame_id_ = acked_frame_id; UpdateFramesInFlight(); } void VideoSender::UpdateFramesInFlight() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (last_sent_frame_id_ != -1) { DCHECK_LE(0, last_sent_frame_id_); uint32 frames_in_flight; if (last_acked_frame_id_ != -1) { DCHECK_LE(0, last_acked_frame_id_); frames_in_flight = static_cast<uint32>(last_sent_frame_id_) - static_cast<uint32>(last_acked_frame_id_); } else { frames_in_flight = static_cast<uint32>(last_sent_frame_id_) + 1; } VLOG(1) << "Frames in flight; last sent: " << last_sent_frame_id_ << " last acked:" << last_acked_frame_id_; if (frames_in_flight >= max_unacked_frames_) { video_encoder_controller_->SkipNextFrame(true); return; } } video_encoder_controller_->SkipNextFrame(false); } void VideoSender::ResendFrame(uint32 resend_frame_id) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); MissingFramesAndPacketsMap missing_frames_and_packets; PacketIdSet missing; missing_frames_and_packets.insert(std::make_pair(resend_frame_id, missing)); rtp_sender_->ResendPackets(missing_frames_and_packets); last_send_time_ = cast_environment_->Clock()->NowTicks(); } } // namespace cast } // namespace media