// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "remoting/client/rectangle_update_decoder.h" #include "base/bind.h" #include "base/callback.h" #include "base/callback_helpers.h" #include "base/location.h" #include "base/logging.h" #include "base/single_thread_task_runner.h" #include "remoting/base/util.h" #include "remoting/client/frame_consumer.h" #include "remoting/codec/video_decoder.h" #include "remoting/codec/video_decoder_verbatim.h" #include "remoting/codec/video_decoder_vpx.h" #include "remoting/protocol/session_config.h" #include "third_party/libyuv/include/libyuv/convert_argb.h" #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" using base::Passed; using remoting::protocol::ChannelConfig; using remoting::protocol::SessionConfig; namespace remoting { // This class wraps a VideoDecoder and byte-swaps the pixels for compatibility // with the android.graphics.Bitmap class. // TODO(lambroslambrou): Refactor so that the VideoDecoder produces data // in the right byte-order, instead of swapping it here. class RgbToBgrVideoDecoderFilter : public VideoDecoder { public: RgbToBgrVideoDecoderFilter(scoped_ptr<VideoDecoder> parent) : parent_(parent.Pass()) { } virtual void Initialize(const webrtc::DesktopSize& screen_size) OVERRIDE { parent_->Initialize(screen_size); } virtual bool DecodePacket(const VideoPacket& packet) OVERRIDE { return parent_->DecodePacket(packet); } virtual void Invalidate(const webrtc::DesktopSize& view_size, const webrtc::DesktopRegion& region) OVERRIDE { return parent_->Invalidate(view_size, region); } virtual void RenderFrame(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, uint8* image_buffer, int image_stride, webrtc::DesktopRegion* output_region) OVERRIDE { parent_->RenderFrame(view_size, clip_area, image_buffer, image_stride, output_region); for (webrtc::DesktopRegion::Iterator i(*output_region); !i.IsAtEnd(); i.Advance()) { webrtc::DesktopRect rect = i.rect(); uint8* pixels = image_buffer + (rect.top() * image_stride) + (rect.left() * kBytesPerPixel); libyuv::ABGRToARGB(pixels, image_stride, pixels, image_stride, rect.width(), rect.height()); } } virtual const webrtc::DesktopRegion* GetImageShape() OVERRIDE { return parent_->GetImageShape(); } private: scoped_ptr<VideoDecoder> parent_; }; RectangleUpdateDecoder::RectangleUpdateDecoder( scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, scoped_refptr<FrameConsumerProxy> consumer) : main_task_runner_(main_task_runner), decode_task_runner_(decode_task_runner), consumer_(consumer), paint_scheduled_(false), latest_sequence_number_(0) { } RectangleUpdateDecoder::~RectangleUpdateDecoder() { } void RectangleUpdateDecoder::Initialize(const SessionConfig& config) { if (!decode_task_runner_->BelongsToCurrentThread()) { decode_task_runner_->PostTask( FROM_HERE, base::Bind(&RectangleUpdateDecoder::Initialize, this, config)); return; } // Initialize decoder based on the selected codec. ChannelConfig::Codec codec = config.video_config().codec; if (codec == ChannelConfig::CODEC_VERBATIM) { decoder_.reset(new VideoDecoderVerbatim()); } else if (codec == ChannelConfig::CODEC_VP8) { decoder_ = VideoDecoderVpx::CreateForVP8(); } else if (codec == ChannelConfig::CODEC_VP9) { decoder_ = VideoDecoderVpx::CreateForVP9(); } else { NOTREACHED() << "Invalid Encoding found: " << codec; } if (consumer_->GetPixelFormat() == FrameConsumer::FORMAT_RGBA) { scoped_ptr<VideoDecoder> wrapper( new RgbToBgrVideoDecoderFilter(decoder_.Pass())); decoder_ = wrapper.Pass(); } } void RectangleUpdateDecoder::DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done) { DCHECK(decode_task_runner_->BelongsToCurrentThread()); base::ScopedClosureRunner done_runner(done); bool decoder_needs_reset = false; bool notify_size_or_dpi_change = false; // If the packet includes screen size or DPI information, store them. if (packet->format().has_screen_width() && packet->format().has_screen_height()) { webrtc::DesktopSize source_size(packet->format().screen_width(), packet->format().screen_height()); if (!source_size_.equals(source_size)) { source_size_ = source_size; decoder_needs_reset = true; notify_size_or_dpi_change = true; } } if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) { webrtc::DesktopVector source_dpi(packet->format().x_dpi(), packet->format().y_dpi()); if (!source_dpi.equals(source_dpi_)) { source_dpi_ = source_dpi; notify_size_or_dpi_change = true; } } // If we've never seen a screen size, ignore the packet. if (source_size_.is_empty()) return; if (decoder_needs_reset) decoder_->Initialize(source_size_); if (notify_size_or_dpi_change) consumer_->SetSourceSize(source_size_, source_dpi_); if (decoder_->DecodePacket(*packet.get())) { SchedulePaint(); } else { LOG(ERROR) << "DecodePacket() failed."; } } void RectangleUpdateDecoder::SchedulePaint() { if (paint_scheduled_) return; paint_scheduled_ = true; decode_task_runner_->PostTask( FROM_HERE, base::Bind(&RectangleUpdateDecoder::DoPaint, this)); } void RectangleUpdateDecoder::DoPaint() { DCHECK(paint_scheduled_); paint_scheduled_ = false; // If the view size is empty or we have no output buffers ready, return. if (buffers_.empty() || view_size_.is_empty()) return; // If no Decoder is initialized, or the host dimensions are empty, return. if (!decoder_.get() || source_size_.is_empty()) return; // Draw the invalidated region to the buffer. webrtc::DesktopFrame* buffer = buffers_.front(); webrtc::DesktopRegion output_region; decoder_->RenderFrame(view_size_, clip_area_, buffer->data(), buffer->stride(), &output_region); // Notify the consumer that painting is done. if (!output_region.is_empty()) { buffers_.pop_front(); consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region); } } void RectangleUpdateDecoder::RequestReturnBuffers(const base::Closure& done) { if (!decode_task_runner_->BelongsToCurrentThread()) { decode_task_runner_->PostTask( FROM_HERE, base::Bind(&RectangleUpdateDecoder::RequestReturnBuffers, this, done)); return; } while (!buffers_.empty()) { consumer_->ReturnBuffer(buffers_.front()); buffers_.pop_front(); } if (!done.is_null()) done.Run(); } void RectangleUpdateDecoder::DrawBuffer(webrtc::DesktopFrame* buffer) { if (!decode_task_runner_->BelongsToCurrentThread()) { decode_task_runner_->PostTask( FROM_HERE, base::Bind(&RectangleUpdateDecoder::DrawBuffer, this, buffer)); return; } DCHECK(clip_area_.width() <= buffer->size().width() && clip_area_.height() <= buffer->size().height()); buffers_.push_back(buffer); SchedulePaint(); } void RectangleUpdateDecoder::InvalidateRegion( const webrtc::DesktopRegion& region) { if (!decode_task_runner_->BelongsToCurrentThread()) { decode_task_runner_->PostTask( FROM_HERE, base::Bind(&RectangleUpdateDecoder::InvalidateRegion, this, region)); return; } if (decoder_.get()) { decoder_->Invalidate(view_size_, region); SchedulePaint(); } } void RectangleUpdateDecoder::SetOutputSizeAndClip( const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area) { if (!decode_task_runner_->BelongsToCurrentThread()) { decode_task_runner_->PostTask( FROM_HERE, base::Bind(&RectangleUpdateDecoder::SetOutputSizeAndClip, this, view_size, clip_area)); return; } // The whole frame needs to be repainted if the scaling factor has changed. if (!view_size_.equals(view_size) && decoder_.get()) { webrtc::DesktopRegion region; region.AddRect(webrtc::DesktopRect::MakeSize(view_size)); decoder_->Invalidate(view_size, region); } if (!view_size_.equals(view_size) || !clip_area_.equals(clip_area)) { view_size_ = view_size; clip_area_ = clip_area; // Return buffers that are smaller than needed to the consumer for // reuse/reallocation. std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin(); while (i != buffers_.end()) { if ((*i)->size().width() < clip_area_.width() || (*i)->size().height() < clip_area_.height()) { consumer_->ReturnBuffer(*i); i = buffers_.erase(i); } else { ++i; } } SchedulePaint(); } } const webrtc::DesktopRegion* RectangleUpdateDecoder::GetBufferShape() { return decoder_->GetImageShape(); } void RectangleUpdateDecoder::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, const base::Closure& done) { DCHECK(main_task_runner_->BelongsToCurrentThread()); // If the video packet is empty then drop it. Empty packets are used to // maintain activity on the network. if (!packet->has_data() || packet->data().size() == 0) { done.Run(); return; } // Add one frame to the counter. stats_.video_frame_rate()->Record(1); // Record other statistics received from host. stats_.video_bandwidth()->Record(packet->data().size()); if (packet->has_capture_time_ms()) stats_.video_capture_ms()->Record(packet->capture_time_ms()); if (packet->has_encode_time_ms()) stats_.video_encode_ms()->Record(packet->encode_time_ms()); if (packet->has_client_sequence_number() && packet->client_sequence_number() > latest_sequence_number_) { latest_sequence_number_ = packet->client_sequence_number(); base::TimeDelta round_trip_latency = base::Time::Now() - base::Time::FromInternalValue(packet->client_sequence_number()); stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds()); } // Measure the latency between the last packet being received and presented. base::Time decode_start = base::Time::Now(); base::Closure decode_done = base::Bind( &RectangleUpdateDecoder::OnPacketDone, this, decode_start, done); decode_task_runner_->PostTask(FROM_HERE, base::Bind( &RectangleUpdateDecoder::DecodePacket, this, base::Passed(&packet), decode_done)); } void RectangleUpdateDecoder::OnPacketDone(base::Time decode_start, const base::Closure& done) { if (!main_task_runner_->BelongsToCurrentThread()) { main_task_runner_->PostTask(FROM_HERE, base::Bind( &RectangleUpdateDecoder::OnPacketDone, this, decode_start, done)); return; } // Record the latency between the packet being received and presented. stats_.video_decode_ms()->Record( (base::Time::Now() - decode_start).InMilliseconds()); done.Run(); } ChromotingStats* RectangleUpdateDecoder::GetStats() { DCHECK(main_task_runner_->BelongsToCurrentThread()); return &stats_; } } // namespace remoting