import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

dom/media/mediasink/AudioSink.h

@@ -0,0 +1,72 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(AudioSink_h__)
+#define AudioSink_h__
+
+#include "mozilla/MozPromise.h"
+#include "mozilla/RefPtr.h"
+#include "nsISupportsImpl.h"
+
+#include "MediaSink.h"
+
+namespace mozilla {
+
+class MediaData;
+template <class T> class MediaQueue;
+
+namespace media {
+
+/*
+ * Define basic APIs for derived class instance to operate or obtain
+ * information from it.
+ */
+class AudioSink {
+public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioSink)
+  AudioSink(MediaQueue<MediaData>& aAudioQueue)
+    : mAudioQueue(aAudioQueue)
+  {}
+
+  typedef MediaSink::PlaybackParams PlaybackParams;
+
+  // Return a promise which will be resolved when AudioSink finishes playing,
+  // or rejected if any error.
+  virtual RefPtr<GenericPromise> Init(const PlaybackParams& aParams) = 0;
+
+  virtual int64_t GetEndTime() const = 0;
+  virtual int64_t GetPosition() = 0;
+
+  // Check whether we've pushed more frames to the audio
+  // hardware than it has played.
+  virtual bool HasUnplayedFrames() = 0;
+
+  // Shut down the AudioSink's resources.
+  virtual void Shutdown() = 0;
+
+  // Change audio playback setting.
+  virtual void SetVolume(double aVolume) = 0;
+  virtual void SetPlaybackRate(double aPlaybackRate) = 0;
+  virtual void SetPreservesPitch(bool aPreservesPitch) = 0;
+
+  // Change audio playback status pause/resume.
+  virtual void SetPlaying(bool aPlaying) = 0;
+
+protected:
+  virtual ~AudioSink() {}
+
+  virtual MediaQueue<MediaData>& AudioQueue() const {
+    return mAudioQueue;
+  }
+
+  // To queue audio data (no matter it's plain or encoded or encrypted, depends
+  // on the subclass)
+  MediaQueue<MediaData>& mAudioQueue;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif

dom/media/mediasink/AudioSinkWrapper.cpp

@@ -0,0 +1,248 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AudioSink.h"
+#include "AudioSinkWrapper.h"
+
+namespace mozilla {
+namespace media {
+
+AudioSinkWrapper::~AudioSinkWrapper()
+{
+}
+
+void
+AudioSinkWrapper::Shutdown()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped.");
+  mCreator = nullptr;
+}
+
+const MediaSink::PlaybackParams&
+AudioSinkWrapper::GetPlaybackParams() const
+{
+  AssertOwnerThread();
+  return mParams;
+}
+
+void
+AudioSinkWrapper::SetPlaybackParams(const PlaybackParams& aParams)
+{
+  AssertOwnerThread();
+  if (mAudioSink) {
+    mAudioSink->SetVolume(aParams.mVolume);
+    mAudioSink->SetPlaybackRate(aParams.mPlaybackRate);
+    mAudioSink->SetPreservesPitch(aParams.mPreservesPitch);
+  }
+  mParams = aParams;
+}
+
+RefPtr<GenericPromise>
+AudioSinkWrapper::OnEnded(TrackType aType)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+  if (aType == TrackInfo::kAudioTrack) {
+    return mEndPromise;
+  }
+  return nullptr;
+}
+
+int64_t
+AudioSinkWrapper::GetEndTime(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+  if (aType == TrackInfo::kAudioTrack && mAudioSink) {
+    return mAudioSink->GetEndTime();
+  }
+  return -1;
+}
+
+int64_t
+AudioSinkWrapper::GetVideoPosition(TimeStamp aNow) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mPlayStartTime.IsNull());
+  // Time elapsed since we started playing.
+  int64_t delta = (aNow - mPlayStartTime).ToMicroseconds();
+  // Take playback rate into account.
+  return mPlayDuration + delta * mParams.mPlaybackRate;
+}
+
+int64_t
+AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+
+  int64_t pos = -1;
+  TimeStamp t = TimeStamp::Now();
+
+  if (!mAudioEnded) {
+    // Rely on the audio sink to report playback position when it is not ended.
+    pos = mAudioSink->GetPosition();
+  } else if (!mPlayStartTime.IsNull()) {
+    // Calculate playback position using system clock if we are still playing.
+    pos = GetVideoPosition(t);
+  } else {
+    // Return how long we've played if we are not playing.
+    pos = mPlayDuration;
+  }
+
+  if (aTimeStamp) {
+    *aTimeStamp = t;
+  }
+
+  return pos;
+}
+
+bool
+AudioSinkWrapper::HasUnplayedFrames(TrackType aType) const
+{
+  AssertOwnerThread();
+  return mAudioSink ? mAudioSink->HasUnplayedFrames() : false;
+}
+
+void
+AudioSinkWrapper::SetVolume(double aVolume)
+{
+  AssertOwnerThread();
+  mParams.mVolume = aVolume;
+  if (mAudioSink) {
+    mAudioSink->SetVolume(aVolume);
+  }
+}
+
+void
+AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate)
+{
+  AssertOwnerThread();
+  if (!mAudioEnded) {
+    // Pass the playback rate to the audio sink. The underlying AudioStream
+    // will handle playback rate changes and report correct audio position.
+    mAudioSink->SetPlaybackRate(aPlaybackRate);
+  } else if (!mPlayStartTime.IsNull()) {
+    // Adjust playback duration and start time when we are still playing.
+    TimeStamp now = TimeStamp::Now();
+    mPlayDuration = GetVideoPosition(now);
+    mPlayStartTime = now;
+  }
+  // mParams.mPlaybackRate affects GetVideoPosition(). It should be updated
+  // after the calls to GetVideoPosition();
+  mParams.mPlaybackRate = aPlaybackRate;
+
+  // Do nothing when not playing. Changes in playback rate will be taken into
+  // account by GetVideoPosition().
+}
+
+void
+AudioSinkWrapper::SetPreservesPitch(bool aPreservesPitch)
+{
+  AssertOwnerThread();
+  mParams.mPreservesPitch = aPreservesPitch;
+  if (mAudioSink) {
+    mAudioSink->SetPreservesPitch(aPreservesPitch);
+  }
+}
+
+void
+AudioSinkWrapper::SetPlaying(bool aPlaying)
+{
+  AssertOwnerThread();
+
+  // Resume/pause matters only when playback started.
+  if (!mIsStarted) {
+    return;
+  }
+
+  if (mAudioSink) {
+    mAudioSink->SetPlaying(aPlaying);
+  }
+
+  if (aPlaying) {
+    MOZ_ASSERT(mPlayStartTime.IsNull());
+    mPlayStartTime = TimeStamp::Now();
+  } else {
+    // Remember how long we've played.
+    mPlayDuration = GetPosition();
+    // mPlayStartTime must be updated later since GetPosition()
+    // depends on the value of mPlayStartTime.
+    mPlayStartTime = TimeStamp();
+  }
+}
+
+void
+AudioSinkWrapper::Start(int64_t aStartTime, const MediaInfo& aInfo)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mIsStarted, "playback already started.");
+
+  mIsStarted = true;
+  mPlayDuration = aStartTime;
+  mPlayStartTime = TimeStamp::Now();
+
+  // no audio is equivalent to audio ended before video starts.
+  mAudioEnded = !aInfo.HasAudio();
+
+  if (aInfo.HasAudio()) {
+    mAudioSink = mCreator->Create();
+    mEndPromise = mAudioSink->Init(mParams);
+
+    mAudioSinkPromise.Begin(mEndPromise->Then(
+      mOwnerThread.get(), __func__, this,
+      &AudioSinkWrapper::OnAudioEnded,
+      &AudioSinkWrapper::OnAudioEnded));
+  }
+}
+
+void
+AudioSinkWrapper::Stop()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "playback not started.");
+
+  mIsStarted = false;
+  mAudioEnded = true;
+
+  if (mAudioSink) {
+    mAudioSinkPromise.DisconnectIfExists();
+    mAudioSink->Shutdown();
+    mAudioSink = nullptr;
+    mEndPromise = nullptr;
+  }
+}
+
+bool
+AudioSinkWrapper::IsStarted() const
+{
+  AssertOwnerThread();
+  return mIsStarted;
+}
+
+bool
+AudioSinkWrapper::IsPlaying() const
+{
+  AssertOwnerThread();
+  return IsStarted() && !mPlayStartTime.IsNull();
+}
+
+void
+AudioSinkWrapper::OnAudioEnded()
+{
+  AssertOwnerThread();
+  mAudioSinkPromise.Complete();
+  mPlayDuration = GetPosition();
+  if (!mPlayStartTime.IsNull()) {
+    mPlayStartTime = TimeStamp::Now();
+  }
+  mAudioEnded = true;
+}
+
+} // namespace media
+} // namespace mozilla
+

dom/media/mediasink/AudioSinkWrapper.h

@@ -0,0 +1,108 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef AudioSinkWrapper_h_
+#define AudioSinkWrapper_h_
+
+#include "mozilla/AbstractThread.h"
+#include "mozilla/dom/AudioChannelBinding.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/TimeStamp.h"
+#include "mozilla/UniquePtr.h"
+
+#include "MediaSink.h"
+
+namespace mozilla {
+
+class MediaData;
+template <class T> class MediaQueue;
+
+namespace media {
+
+class AudioSink;
+
+/**
+ * A wrapper around AudioSink to provide the interface of MediaSink.
+ */
+class AudioSinkWrapper : public MediaSink {
+  // An AudioSink factory.
+  class Creator {
+  public:
+    virtual ~Creator() {}
+    virtual AudioSink* Create() = 0;
+  };
+
+  // Wrap around a function object which creates AudioSinks.
+  template <typename Function>
+  class CreatorImpl : public Creator {
+  public:
+    explicit CreatorImpl(const Function& aFunc) : mFunction(aFunc) {}
+    AudioSink* Create() override { return mFunction(); }
+  private:
+    Function mFunction;
+  };
+
+public:
+  template <typename Function>
+  AudioSinkWrapper(AbstractThread* aOwnerThread, const Function& aFunc)
+    : mOwnerThread(aOwnerThread)
+    , mCreator(new CreatorImpl<Function>(aFunc))
+    , mIsStarted(false)
+    // Give an insane value to facilitate debug if used before playback starts.
+    , mPlayDuration(INT64_MAX)
+    , mAudioEnded(true)
+  {}
+
+  const PlaybackParams& GetPlaybackParams() const override;
+  void SetPlaybackParams(const PlaybackParams& aParams) override;
+
+  RefPtr<GenericPromise> OnEnded(TrackType aType) override;
+  int64_t GetEndTime(TrackType aType) const override;
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
+  bool HasUnplayedFrames(TrackType aType) const override;
+
+  void SetVolume(double aVolume) override;
+  void SetPlaybackRate(double aPlaybackRate) override;
+  void SetPreservesPitch(bool aPreservesPitch) override;
+  void SetPlaying(bool aPlaying) override;
+
+  void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
+  void Stop() override;
+  bool IsStarted() const override;
+  bool IsPlaying() const override;
+
+  void Shutdown() override;
+
+private:
+  virtual ~AudioSinkWrapper();
+
+  void AssertOwnerThread() const {
+    MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+  }
+
+  int64_t GetVideoPosition(TimeStamp aNow) const;
+
+  void OnAudioEnded();
+
+  const RefPtr<AbstractThread> mOwnerThread;
+  UniquePtr<Creator> mCreator;
+  RefPtr<AudioSink> mAudioSink;
+  RefPtr<GenericPromise> mEndPromise;
+
+  bool mIsStarted;
+  PlaybackParams mParams;
+
+  TimeStamp mPlayStartTime;
+  int64_t mPlayDuration;
+
+  bool mAudioEnded;
+  MozPromiseRequestHolder<GenericPromise> mAudioSinkPromise;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif //AudioSinkWrapper_h_

dom/media/mediasink/DecodedAudioDataSink.cpp

@@ -0,0 +1,561 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsPrintfCString.h"
+#include "MediaQueue.h"
+#include "DecodedAudioDataSink.h"
+#include "VideoUtils.h"
+#include "AudioConverter.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/DebugOnly.h"
+#include "MediaPrefs.h"
+
+namespace mozilla {
+
+extern LazyLogModule gMediaDecoderLog;
+#define SINK_LOG(msg, ...) \
+  MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, \
+    ("DecodedAudioDataSink=%p " msg, this, ##__VA_ARGS__))
+#define SINK_LOG_V(msg, ...) \
+  MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, \
+  ("DecodedAudioDataSink=%p " msg, this, ##__VA_ARGS__))
+
+namespace media {
+
+// The amount of audio frames that is used to fuzz rounding errors.
+static const int64_t AUDIO_FUZZ_FRAMES = 1;
+
+// Amount of audio frames we will be processing ahead of use
+static const int32_t LOW_AUDIO_USECS = 300000;
+
+DecodedAudioDataSink::DecodedAudioDataSink(AbstractThread* aThread,
+                                           MediaQueue<MediaData>& aAudioQueue,
+                                           int64_t aStartTime,
+                                           const AudioInfo& aInfo,
+                                           dom::AudioChannel aChannel)
+  : AudioSink(aAudioQueue)
+  , mStartTime(aStartTime)
+  , mLastGoodPosition(0)
+  , mInfo(aInfo)
+  , mChannel(aChannel)
+  , mPlaying(true)
+  , mMonitor("DecodedAudioDataSink")
+  , mWritten(0)
+  , mErrored(false)
+  , mPlaybackComplete(false)
+  , mOwnerThread(aThread)
+  , mProcessedQueueLength(0)
+  , mFramesParsed(0)
+  , mLastEndTime(0)
+  , mIsAudioDataAudible(false)
+{
+  bool resampling = MediaPrefs::AudioSinkResampling();
+
+  if (resampling) {
+    mOutputRate = MediaPrefs::AudioSinkResampleRate();
+  } else if (mInfo.mRate == 44100 || mInfo.mRate == 48000) {
+    // The original rate is of good quality and we want to minimize unecessary
+    // resampling. The common scenario being that the sampling rate is one or
+    // the other, this allows to minimize audio quality regression and hoping
+    // content provider want change from those rates mid-stream.
+    mOutputRate = mInfo.mRate;
+  } else {
+    // We will resample all data to match cubeb's preferred sampling rate.
+    mOutputRate = AudioStream::GetPreferredRate();
+  }
+  MOZ_DIAGNOSTIC_ASSERT(mOutputRate, "output rate can't be 0.");
+
+  bool monoAudioEnabled = MediaPrefs::MonoAudio();
+
+  mOutputChannels = monoAudioEnabled
+    ? 1 : (MediaPrefs::AudioSinkForceStereo() ? 2 : mInfo.mChannels);
+}
+
+DecodedAudioDataSink::~DecodedAudioDataSink()
+{
+}
+
+RefPtr<GenericPromise>
+DecodedAudioDataSink::Init(const PlaybackParams& aParams)
+{
+  MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+
+  mAudioQueueListener = mAudioQueue.PushEvent().Connect(
+    mOwnerThread, this, &DecodedAudioDataSink::OnAudioPushed);
+  mAudioQueueFinishListener = mAudioQueue.FinishEvent().Connect(
+    mOwnerThread, this, &DecodedAudioDataSink::NotifyAudioNeeded);
+  mProcessedQueueListener = mProcessedQueue.PopEvent().Connect(
+    mOwnerThread, this, &DecodedAudioDataSink::OnAudioPopped);
+
+  // To ensure at least one audio packet will be popped from AudioQueue and
+  // ready to be played.
+  NotifyAudioNeeded();
+  RefPtr<GenericPromise> p = mEndPromise.Ensure(__func__);
+  nsresult rv = InitializeAudioStream(aParams);
+  if (NS_FAILED(rv)) {
+    mEndPromise.Reject(rv, __func__);
+  }
+  return p;
+}
+
+int64_t
+DecodedAudioDataSink::GetPosition()
+{
+  int64_t pos;
+  if (mAudioStream &&
+      (pos = mAudioStream->GetPosition()) >= 0) {
+    NS_ASSERTION(pos >= mLastGoodPosition,
+                 "AudioStream position shouldn't go backward");
+    // Update the last good position when we got a good one.
+    if (pos >= mLastGoodPosition) {
+      mLastGoodPosition = pos;
+    }
+  }
+
+  return mStartTime + mLastGoodPosition;
+}
+
+bool
+DecodedAudioDataSink::HasUnplayedFrames()
+{
+  // Experimentation suggests that GetPositionInFrames() is zero-indexed,
+  // so we need to add 1 here before comparing it to mWritten.
+  int64_t total;
+  {
+    MonitorAutoLock mon(mMonitor);
+    total = mWritten + (mCursor.get() ? mCursor->Available() : 0);
+  }
+  return mProcessedQueue.GetSize() ||
+         (mAudioStream && mAudioStream->GetPositionInFrames() + 1 < total);
+}
+
+void
+DecodedAudioDataSink::Shutdown()
+{
+  MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+
+  mAudioQueueListener.Disconnect();
+  mAudioQueueFinishListener.Disconnect();
+  mProcessedQueueListener.Disconnect();
+
+  if (mAudioStream) {
+    mAudioStream->Shutdown();
+    mAudioStream = nullptr;
+  }
+  mProcessedQueue.Reset();
+  mProcessedQueue.Finish();
+  mEndPromise.ResolveIfExists(true, __func__);
+}
+
+void
+DecodedAudioDataSink::SetVolume(double aVolume)
+{
+  if (mAudioStream) {
+    mAudioStream->SetVolume(aVolume);
+  }
+}
+
+void
+DecodedAudioDataSink::SetPlaybackRate(double aPlaybackRate)
+{
+  MOZ_ASSERT(aPlaybackRate != 0, "Don't set the playbackRate to 0 on AudioStream");
+  if (mAudioStream) {
+    mAudioStream->SetPlaybackRate(aPlaybackRate);
+  }
+}
+
+void
+DecodedAudioDataSink::SetPreservesPitch(bool aPreservesPitch)
+{
+  if (mAudioStream) {
+    mAudioStream->SetPreservesPitch(aPreservesPitch);
+  }
+}
+
+void
+DecodedAudioDataSink::SetPlaying(bool aPlaying)
+{
+  if (!mAudioStream || mPlaying == aPlaying || mPlaybackComplete) {
+    return;
+  }
+  // pause/resume AudioStream as necessary.
+  if (!aPlaying) {
+    mAudioStream->Pause();
+  } else if (aPlaying) {
+    mAudioStream->Resume();
+  }
+  mPlaying = aPlaying;
+}
+
+nsresult
+DecodedAudioDataSink::InitializeAudioStream(const PlaybackParams& aParams)
+{
+  mAudioStream = new AudioStream(*this);
+  nsresult rv = mAudioStream->Init(mOutputChannels, mOutputRate, mChannel);
+  if (NS_FAILED(rv)) {
+    mAudioStream->Shutdown();
+    mAudioStream = nullptr;
+    return rv;
+  }
+
+  // Set playback params before calling Start() so they can take effect
+  // as soon as the 1st DataCallback of the AudioStream fires.
+  mAudioStream->SetVolume(aParams.mVolume);
+  mAudioStream->SetPlaybackRate(aParams.mPlaybackRate);
+  mAudioStream->SetPreservesPitch(aParams.mPreservesPitch);
+  mAudioStream->Start();
+
+  return NS_OK;
+}
+
+int64_t
+DecodedAudioDataSink::GetEndTime() const
+{
+  int64_t written;
+  {
+    MonitorAutoLock mon(mMonitor);
+    written = mWritten;
+  }
+  CheckedInt64 playedUsecs = FramesToUsecs(written, mOutputRate) + mStartTime;
+  if (!playedUsecs.isValid()) {
+    NS_WARNING("Int overflow calculating audio end time");
+    return -1;
+  }
+  // As we may be resampling, rounding errors may occur. Ensure we never get
+  // past the original end time.
+  return std::min<int64_t>(mLastEndTime, playedUsecs.value());
+}
+
+UniquePtr<AudioStream::Chunk>
+DecodedAudioDataSink::PopFrames(uint32_t aFrames)
+{
+  class Chunk : public AudioStream::Chunk {
+  public:
+    Chunk(AudioData* aBuffer, uint32_t aFrames, AudioDataValue* aData)
+      : mBuffer(aBuffer), mFrames(aFrames), mData(aData) {}
+    Chunk() : mFrames(0), mData(nullptr) {}
+    const AudioDataValue* Data() const { return mData; }
+    uint32_t Frames() const { return mFrames; }
+    uint32_t Channels() const { return mBuffer ? mBuffer->mChannels: 0; }
+    uint32_t Rate() const { return mBuffer ? mBuffer->mRate : 0; }
+    AudioDataValue* GetWritable() const { return mData; }
+  private:
+    const RefPtr<AudioData> mBuffer;
+    const uint32_t mFrames;
+    AudioDataValue* const mData;
+  };
+
+  class SilentChunk : public AudioStream::Chunk {
+  public:
+    SilentChunk(uint32_t aFrames, uint32_t aChannels, uint32_t aRate)
+      : mFrames(aFrames)
+      , mChannels(aChannels)
+      , mRate(aRate)
+      , mData(MakeUnique<AudioDataValue[]>(aChannels * aFrames)) {
+      memset(mData.get(), 0, aChannels * aFrames * sizeof(AudioDataValue));
+    }
+    const AudioDataValue* Data() const { return mData.get(); }
+    uint32_t Frames() const { return mFrames; }
+    uint32_t Channels() const { return mChannels; }
+    uint32_t Rate() const { return mRate; }
+    AudioDataValue* GetWritable() const { return mData.get(); }
+  private:
+    const uint32_t mFrames;
+    const uint32_t mChannels;
+    const uint32_t mRate;
+    UniquePtr<AudioDataValue[]> mData;
+  };
+
+  bool needPopping = false;
+  if (!mCurrentData) {
+    // No data in the queue. Return an empty chunk.
+    if (!mProcessedQueue.GetSize()) {
+      return MakeUnique<Chunk>();
+    }
+
+    // We need to update our values prior popping the processed queue in
+    // order to prevent the pop event to fire too early (prior
+    // mProcessedQueueLength being updated) or prevent HasUnplayedFrames
+    // to incorrectly return true during the time interval betweeen the
+    // when mProcessedQueue is read and mWritten is updated.
+    needPopping = true;
+    mCurrentData = mProcessedQueue.PeekFront();
+    {
+      MonitorAutoLock mon(mMonitor);
+      mCursor = MakeUnique<AudioBufferCursor>(mCurrentData->mAudioData.get(),
+                                              mCurrentData->mChannels,
+                                              mCurrentData->mFrames);
+    }
+    MOZ_ASSERT(mCurrentData->mFrames > 0);
+    mProcessedQueueLength -=
+      FramesToUsecs(mCurrentData->mFrames, mOutputRate).value();
+  }
+
+  auto framesToPop = std::min(aFrames, mCursor->Available());
+
+  SINK_LOG_V("playing audio at time=%lld offset=%u length=%u",
+             mCurrentData->mTime, mCurrentData->mFrames - mCursor->Available(), framesToPop);
+
+  UniquePtr<AudioStream::Chunk> chunk =
+    MakeUnique<Chunk>(mCurrentData, framesToPop, mCursor->Ptr());
+
+  {
+    MonitorAutoLock mon(mMonitor);
+    mWritten += framesToPop;
+    mCursor->Advance(framesToPop);
+  }
+
+  // All frames are popped. Reset mCurrentData so we can pop new elements from
+  // the audio queue in next calls to PopFrames().
+  if (!mCursor->Available()) {
+    mCurrentData = nullptr;
+  }
+
+  if (needPopping) {
+    // We can now safely pop the audio packet from the processed queue.
+    // This will fire the popped event, triggering a call to NotifyAudioNeeded.
+    RefPtr<AudioData> releaseMe = mProcessedQueue.PopFront();
+    CheckIsAudible(releaseMe);
+  }
+
+  return chunk;
+}
+
+bool
+DecodedAudioDataSink::Ended() const
+{
+  // Return true when error encountered so AudioStream can start draining.
+  return mProcessedQueue.IsFinished() || mErrored;
+}
+
+void
+DecodedAudioDataSink::Drained()
+{
+  SINK_LOG("Drained");
+  mPlaybackComplete = true;
+  mEndPromise.ResolveIfExists(true, __func__);
+}
+
+void
+DecodedAudioDataSink::CheckIsAudible(const AudioData* aData)
+{
+  MOZ_ASSERT(aData);
+
+  bool isAudible = aData->IsAudible();
+  if (isAudible != mIsAudioDataAudible) {
+    mIsAudioDataAudible = isAudible;
+    mAudibleEvent.Notify(mIsAudioDataAudible);
+  }
+}
+
+void
+DecodedAudioDataSink::OnAudioPopped(const RefPtr<MediaData>& aSample)
+{
+  SINK_LOG_V("AudioStream has used an audio packet.");
+  NotifyAudioNeeded();
+}
+
+void
+DecodedAudioDataSink::OnAudioPushed(const RefPtr<MediaData>& aSample)
+{
+  SINK_LOG_V("One new audio packet available.");
+  NotifyAudioNeeded();
+}
+
+void
+DecodedAudioDataSink::NotifyAudioNeeded()
+{
+  MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn(),
+             "Not called from the owner's thread");
+
+  // Always ensure we have two processed frames pending to allow for processing
+  // latency.
+  while (AudioQueue().GetSize() && (AudioQueue().IsFinished() ||
+                                    mProcessedQueueLength < LOW_AUDIO_USECS ||
+                                    mProcessedQueue.GetSize() < 2)) {
+    RefPtr<AudioData> data =
+      dont_AddRef(AudioQueue().PopFront().take()->As<AudioData>());
+
+    // Ignore the element with 0 frames and try next.
+    if (!data->mFrames) {
+      continue;
+    }
+
+    if (!mConverter ||
+        (data->mRate != mConverter->InputConfig().Rate() ||
+         data->mChannels != mConverter->InputConfig().Channels())) {
+      SINK_LOG_V("Audio format changed from %u@%uHz to %u@%uHz",
+                 mConverter? mConverter->InputConfig().Channels() : 0,
+                 mConverter ? mConverter->InputConfig().Rate() : 0,
+                 data->mChannels, data->mRate);
+
+      DrainConverter();
+
+      // mFramesParsed indicates the current playtime in frames at the current
+      // input sampling rate. Recalculate it per the new sampling rate.
+      if (mFramesParsed) {
+        // We minimize overflow.
+        uint32_t oldRate = mConverter->InputConfig().Rate();
+        uint32_t newRate = data->mRate;
+        CheckedInt64 result = SaferMultDiv(mFramesParsed, newRate, oldRate);
+        if (!result.isValid()) {
+          NS_WARNING("Int overflow in DecodedAudioDataSink");
+          mErrored = true;
+          return;
+        }
+        mFramesParsed = result.value();
+      }
+
+      mConverter =
+        MakeUnique<AudioConverter>(
+          AudioConfig(data->mChannels, data->mRate),
+          AudioConfig(mOutputChannels, mOutputRate));
+    }
+
+    // See if there's a gap in the audio. If there is, push silence into the
+    // audio hardware, so we can play across the gap.
+    // Calculate the timestamp of the next chunk of audio in numbers of
+    // samples.
+    CheckedInt64 sampleTime = UsecsToFrames(data->mTime - mStartTime,
+                                            data->mRate);
+    // Calculate the number of frames that have been pushed onto the audio hardware.
+    CheckedInt64 missingFrames = sampleTime - mFramesParsed;
+
+    if (!missingFrames.isValid()) {
+      NS_WARNING("Int overflow in DecodedAudioDataSink");
+      mErrored = true;
+      return;
+    }
+
+    if (missingFrames.value() > AUDIO_FUZZ_FRAMES) {
+      // The next audio packet begins some time after the end of the last packet
+      // we pushed to the audio hardware. We must push silence into the audio
+      // hardware so that the next audio packet begins playback at the correct
+      // time.
+      missingFrames = std::min<int64_t>(INT32_MAX, missingFrames.value());
+      mFramesParsed += missingFrames.value();
+
+      // We need to calculate how many frames are missing at the output rate.
+      missingFrames =
+        SaferMultDiv(missingFrames.value(), mOutputRate, data->mRate);
+      if (!missingFrames.isValid()) {
+        NS_WARNING("Int overflow in DecodedAudioDataSink");
+        mErrored = true;
+        return;
+      }
+
+      // We need to insert silence, first use drained frames if any.
+      missingFrames -= DrainConverter(missingFrames.value());
+      // Insert silence if still needed.
+      if (missingFrames.value()) {
+        AlignedAudioBuffer silenceData(missingFrames.value() * mOutputChannels);
+        if (!silenceData) {
+          NS_WARNING("OOM in DecodedAudioDataSink");
+          mErrored = true;
+          return;
+        }
+        RefPtr<AudioData> silence = CreateAudioFromBuffer(Move(silenceData), data);
+        PushProcessedAudio(silence);
+      }
+    }
+
+    mLastEndTime = data->GetEndTime();
+    mFramesParsed += data->mFrames;
+
+    if (mConverter->InputConfig() != mConverter->OutputConfig()) {
+      // We must ensure that the size in the buffer contains exactly the number
+      // of frames, in case one of the audio producer over allocated the buffer.
+      AlignedAudioBuffer buffer(Move(data->mAudioData));
+      buffer.SetLength(size_t(data->mFrames) * data->mChannels);
+
+      AlignedAudioBuffer convertedData =
+        mConverter->Process(AudioSampleBuffer(Move(buffer))).Forget();
+      data = CreateAudioFromBuffer(Move(convertedData), data);
+    }
+    if (PushProcessedAudio(data)) {
+      mLastProcessedPacket = Some(data);
+    }
+  }
+
+  if (AudioQueue().IsFinished()) {
+    // We have reached the end of the data, drain the resampler.
+    DrainConverter();
+    mProcessedQueue.Finish();
+  }
+}
+
+uint32_t
+DecodedAudioDataSink::PushProcessedAudio(AudioData* aData)
+{
+  if (!aData || !aData->mFrames) {
+    return 0;
+  }
+  mProcessedQueue.Push(aData);
+  mProcessedQueueLength += FramesToUsecs(aData->mFrames, mOutputRate).value();
+  return aData->mFrames;
+}
+
+already_AddRefed<AudioData>
+DecodedAudioDataSink::CreateAudioFromBuffer(AlignedAudioBuffer&& aBuffer,
+                                            AudioData* aReference)
+{
+  uint32_t frames = aBuffer.Length() / mOutputChannels;
+  if (!frames) {
+    return nullptr;
+  }
+  CheckedInt64 duration = FramesToUsecs(frames, mOutputRate);
+  if (!duration.isValid()) {
+    NS_WARNING("Int overflow in DecodedAudioDataSink");
+    mErrored = true;
+    return nullptr;
+  }
+  RefPtr<AudioData> data =
+    new AudioData(aReference->mOffset,
+                  aReference->mTime,
+                  duration.value(),
+                  frames,
+                  Move(aBuffer),
+                  mOutputChannels,
+                  mOutputRate);
+  return data.forget();
+}
+
+uint32_t
+DecodedAudioDataSink::DrainConverter(uint32_t aMaxFrames)
+{
+  MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+
+  if (!mConverter || !mLastProcessedPacket || !aMaxFrames) {
+    // nothing to drain.
+    return 0;
+  }
+
+  RefPtr<AudioData> lastPacket = mLastProcessedPacket.ref();
+  mLastProcessedPacket.reset();
+
+  // To drain we simply provide an empty packet to the audio converter.
+  AlignedAudioBuffer convertedData =
+    mConverter->Process(AudioSampleBuffer(AlignedAudioBuffer())).Forget();
+
+  uint32_t frames = convertedData.Length() / mOutputChannels;
+  if (!convertedData.SetLength(std::min(frames, aMaxFrames) * mOutputChannels)) {
+    // This can never happen as we were reducing the length of convertData.
+    mErrored = true;
+    return 0;
+  }
+
+  RefPtr<AudioData> data =
+    CreateAudioFromBuffer(Move(convertedData), lastPacket);
+  if (!data) {
+    return 0;
+  }
+  mProcessedQueue.Push(data);
+  return data->mFrames;
+}
+
+} // namespace media
+} // namespace mozilla

dom/media/mediasink/DecodedAudioDataSink.h

@@ -0,0 +1,165 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(DecodedAudioDataSink_h__)
+#define DecodedAudioDataSink_h__
+
+#include "AudioSink.h"
+#include "AudioStream.h"
+#include "MediaEventSource.h"
+#include "MediaQueue.h"
+#include "MediaInfo.h"
+#include "mozilla/RefPtr.h"
+#include "nsISupportsImpl.h"
+
+#include "mozilla/dom/AudioChannelBinding.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/Monitor.h"
+
+namespace mozilla {
+
+class AudioConverter;
+
+namespace media {
+
+class DecodedAudioDataSink : public AudioSink,
+                             private AudioStream::DataSource {
+public:
+  DecodedAudioDataSink(AbstractThread* aThread,
+                       MediaQueue<MediaData>& aAudioQueue,
+                       int64_t aStartTime,
+                       const AudioInfo& aInfo,
+                       dom::AudioChannel aChannel);
+
+  // Return a promise which will be resolved when DecodedAudioDataSink
+  // finishes playing, or rejected if any error.
+  RefPtr<GenericPromise> Init(const PlaybackParams& aParams) override;
+
+  /*
+   * All public functions are not thread-safe.
+   * Called on the task queue of MDSM only.
+   */
+  int64_t GetPosition() override;
+  int64_t GetEndTime() const override;
+
+  // Check whether we've pushed more frames to the audio hardware than it has
+  // played.
+  bool HasUnplayedFrames() override;
+
+  // Shut down the DecodedAudioDataSink's resources.
+  void Shutdown() override;
+
+  void SetVolume(double aVolume) override;
+  void SetPlaybackRate(double aPlaybackRate) override;
+  void SetPreservesPitch(bool aPreservesPitch) override;
+  void SetPlaying(bool aPlaying) override;
+
+  MediaEventSource<bool>& AudibleEvent() {
+    return mAudibleEvent;
+  }
+
+private:
+  virtual ~DecodedAudioDataSink();
+
+  // Allocate and initialize mAudioStream. Returns NS_OK on success.
+  nsresult InitializeAudioStream(const PlaybackParams& aParams);
+
+  // Interface of AudioStream::DataSource.
+  // Called on the callback thread of cubeb.
+  UniquePtr<AudioStream::Chunk> PopFrames(uint32_t aFrames) override;
+  bool Ended() const override;
+  void Drained() override;
+
+  void CheckIsAudible(const AudioData* aData);
+
+  // The audio stream resource. Used on the task queue of MDSM only.
+  RefPtr<AudioStream> mAudioStream;
+
+  // The presentation time of the first audio frame that was played in
+  // microseconds. We can add this to the audio stream position to determine
+  // the current audio time.
+  const int64_t mStartTime;
+
+  // Keep the last good position returned from the audio stream. Used to ensure
+  // position returned by GetPosition() is mono-increasing in spite of audio
+  // stream error. Used on the task queue of MDSM only.
+  int64_t mLastGoodPosition;
+
+  const AudioInfo mInfo;
+
+  const dom::AudioChannel mChannel;
+
+  // Used on the task queue of MDSM only.
+  bool mPlaying;
+
+  MozPromiseHolder<GenericPromise> mEndPromise;
+
+  /*
+   * Members to implement AudioStream::DataSource.
+   * Used on the callback thread of cubeb.
+   */
+  // The AudioData at which AudioStream::DataSource is reading.
+  RefPtr<AudioData> mCurrentData;
+
+  // Monitor protecting access to mCursor and mWritten.
+  // mCursor is created/destroyed on the cubeb thread, while we must also
+  // ensure that mWritten and mCursor::Available() get modified simultaneously.
+  // (written on cubeb thread, and read on MDSM task queue).
+  mutable Monitor mMonitor;
+  // Keep track of the read position of mCurrentData.
+  UniquePtr<AudioBufferCursor> mCursor;
+
+  // PCM frames written to the stream so far.
+  int64_t mWritten;
+
+  // True if there is any error in processing audio data like overflow.
+  Atomic<bool> mErrored;
+
+  // Set on the callback thread of cubeb once the stream has drained.
+  Atomic<bool> mPlaybackComplete;
+
+  const RefPtr<AbstractThread> mOwnerThread;
+
+  // Audio Processing objects and methods
+  void OnAudioPopped(const RefPtr<MediaData>& aSample);
+  void OnAudioPushed(const RefPtr<MediaData>& aSample);
+  void NotifyAudioNeeded();
+  // Drain the converter and add the output to the processed audio queue.
+  // A maximum of aMaxFrames will be added.
+  uint32_t DrainConverter(uint32_t aMaxFrames = UINT32_MAX);
+  already_AddRefed<AudioData> CreateAudioFromBuffer(AlignedAudioBuffer&& aBuffer,
+                                                    AudioData* aReference);
+  // Add data to the processsed queue, update mProcessedQueueLength and
+  // return the number of frames added.
+  uint32_t PushProcessedAudio(AudioData* aData);
+  UniquePtr<AudioConverter> mConverter;
+  MediaQueue<AudioData> mProcessedQueue;
+  // Length in microseconds of the ProcessedQueue
+  Atomic<int32_t> mProcessedQueueLength;
+  MediaEventListener mAudioQueueListener;
+  MediaEventListener mAudioQueueFinishListener;
+  MediaEventListener mProcessedQueueListener;
+  // Number of frames processed from AudioQueue(). Used to determine gaps in
+  // the input stream. It indicates the time in frames since playback started
+  // at the current input framerate.
+  int64_t mFramesParsed;
+  Maybe<RefPtr<AudioData>> mLastProcessedPacket;
+  int64_t mLastEndTime;
+  // Never modifed after construction.
+  uint32_t mOutputRate;
+  uint32_t mOutputChannels;
+
+  // True when audio is producing audible sound, false when audio is silent.
+  bool mIsAudioDataAudible;
+
+  MediaEventProducer<bool> mAudibleEvent;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif

dom/media/mediasink/DecodedStream.cpp

@@ -0,0 +1,781 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/gfx/Point.h"
+#include "mozilla/SyncRunnable.h"
+
+#include "AudioSegment.h"
+#include "DecodedStream.h"
+#include "MediaData.h"
+#include "MediaQueue.h"
+#include "MediaStreamGraph.h"
+#include "MediaStreamListener.h"
+#include "OutputStreamManager.h"
+#include "SharedBuffer.h"
+#include "VideoSegment.h"
+#include "VideoUtils.h"
+
+namespace mozilla {
+
+#undef DUMP_LOG
+#define DUMP_LOG(x, ...) NS_DebugBreak(NS_DEBUG_WARNING, nsPrintfCString(x, ##__VA_ARGS__).get(), nullptr, nullptr, -1)
+
+/*
+ * A container class to make it easier to pass the playback info all the
+ * way to DecodedStreamGraphListener from DecodedStream.
+ */
+struct PlaybackInfoInit {
+  int64_t mStartTime;
+  MediaInfo mInfo;
+};
+
+class DecodedStreamGraphListener : public MediaStreamListener {
+public:
+  DecodedStreamGraphListener(MediaStream* aStream,
+                             MozPromiseHolder<GenericPromise>&& aPromise)
+    : mMutex("DecodedStreamGraphListener::mMutex")
+    , mStream(aStream)
+  {
+    mFinishPromise = Move(aPromise);
+  }
+
+  void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) override
+  {
+    MutexAutoLock lock(mMutex);
+    if (mStream) {
+      int64_t t = mStream->StreamTimeToMicroseconds(
+        mStream->GraphTimeToStreamTime(aCurrentTime));
+      mOnOutput.Notify(t);
+    }
+  }
+
+  void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent event) override
+  {
+    if (event == MediaStreamGraphEvent::EVENT_FINISHED) {
+      nsCOMPtr<nsIRunnable> event =
+        NewRunnableMethod(this, &DecodedStreamGraphListener::DoNotifyFinished);
+      aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
+    }
+  }
+
+  void DoNotifyFinished()
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+    mFinishPromise.ResolveIfExists(true, __func__);
+  }
+
+  void Forget()
+  {
+    RefPtr<DecodedStreamGraphListener> self = this;
+    AbstractThread::MainThread()->Dispatch(NS_NewRunnableFunction([self] () {
+      MOZ_ASSERT(NS_IsMainThread());
+      self->mFinishPromise.ResolveIfExists(true, __func__);
+    }));
+    MutexAutoLock lock(mMutex);
+    mStream = nullptr;
+  }
+
+  MediaEventSource<int64_t>& OnOutput()
+  {
+    return mOnOutput;
+  }
+
+private:
+  MediaEventProducer<int64_t> mOnOutput;
+
+  Mutex mMutex;
+  // Members below are protected by mMutex.
+  RefPtr<MediaStream> mStream;
+  // Main thread only.
+  MozPromiseHolder<GenericPromise> mFinishPromise;
+};
+
+static void
+UpdateStreamSuspended(MediaStream* aStream, bool aBlocking)
+{
+  if (NS_IsMainThread()) {
+    if (aBlocking) {
+      aStream->Suspend();
+    } else {
+      aStream->Resume();
+    }
+  } else {
+    nsCOMPtr<nsIRunnable> r;
+    if (aBlocking) {
+      r = NewRunnableMethod(aStream, &MediaStream::Suspend);
+    } else {
+      r = NewRunnableMethod(aStream, &MediaStream::Resume);
+    }
+    AbstractThread::MainThread()->Dispatch(r.forget());
+  }
+}
+
+/*
+ * All MediaStream-related data is protected by the decoder's monitor.
+ * We have at most one DecodedStreamDaata per MediaDecoder. Its stream
+ * is used as the input for each ProcessedMediaStream created by calls to
+ * captureStream(UntilEnded). Seeking creates a new source stream, as does
+ * replaying after the input as ended. In the latter case, the new source is
+ * not connected to streams created by captureStreamUntilEnded.
+ */
+class DecodedStreamData {
+public:
+  DecodedStreamData(OutputStreamManager* aOutputStreamManager,
+                    PlaybackInfoInit&& aInit,
+                    MozPromiseHolder<GenericPromise>&& aPromise);
+  ~DecodedStreamData();
+  void SetPlaying(bool aPlaying);
+  MediaEventSource<int64_t>& OnOutput();
+  void Forget();
+  void DumpDebugInfo();
+
+  /* The following group of fields are protected by the decoder's monitor
+   * and can be read or written on any thread.
+   */
+  // Count of audio frames written to the stream
+  int64_t mAudioFramesWritten;
+  // mNextVideoTime is the end timestamp for the last packet sent to the stream.
+  // Therefore video packets starting at or after this time need to be copied
+  // to the output stream.
+  int64_t mNextVideoTime; // microseconds
+  int64_t mNextAudioTime; // microseconds
+  // The last video image sent to the stream. Useful if we need to replicate
+  // the image.
+  RefPtr<layers::Image> mLastVideoImage;
+  gfx::IntSize mLastVideoImageDisplaySize;
+  bool mHaveSentFinish;
+  bool mHaveSentFinishAudio;
+  bool mHaveSentFinishVideo;
+
+  // The decoder is responsible for calling Destroy() on this stream.
+  const RefPtr<SourceMediaStream> mStream;
+  const RefPtr<DecodedStreamGraphListener> mListener;
+  bool mPlaying;
+  // True if we need to send a compensation video frame to ensure the
+  // StreamTime going forward.
+  bool mEOSVideoCompensation;
+
+  const RefPtr<OutputStreamManager> mOutputStreamManager;
+};
+
+DecodedStreamData::DecodedStreamData(OutputStreamManager* aOutputStreamManager,
+                                     PlaybackInfoInit&& aInit,
+                                     MozPromiseHolder<GenericPromise>&& aPromise)
+  : mAudioFramesWritten(0)
+  , mNextVideoTime(aInit.mStartTime)
+  , mNextAudioTime(aInit.mStartTime)
+  , mHaveSentFinish(false)
+  , mHaveSentFinishAudio(false)
+  , mHaveSentFinishVideo(false)
+  , mStream(aOutputStreamManager->Graph()->CreateSourceStream())
+  // DecodedStreamGraphListener will resolve this promise.
+  , mListener(new DecodedStreamGraphListener(mStream, Move(aPromise)))
+  // mPlaying is initially true because MDSM won't start playback until playing
+  // becomes true. This is consistent with the settings of AudioSink.
+  , mPlaying(true)
+  , mEOSVideoCompensation(false)
+  , mOutputStreamManager(aOutputStreamManager)
+{
+  mStream->AddListener(mListener);
+  mOutputStreamManager->Connect(mStream);
+
+  // Initialize tracks.
+  if (aInit.mInfo.HasAudio()) {
+    mStream->AddAudioTrack(aInit.mInfo.mAudio.mTrackId,
+                           aInit.mInfo.mAudio.mRate,
+                           0, new AudioSegment());
+  }
+  if (aInit.mInfo.HasVideo()) {
+    mStream->AddTrack(aInit.mInfo.mVideo.mTrackId, 0, new VideoSegment());
+  }
+}
+
+DecodedStreamData::~DecodedStreamData()
+{
+  mOutputStreamManager->Disconnect();
+  mStream->Destroy();
+}
+
+MediaEventSource<int64_t>&
+DecodedStreamData::OnOutput()
+{
+  return mListener->OnOutput();
+}
+
+void
+DecodedStreamData::SetPlaying(bool aPlaying)
+{
+  if (mPlaying != aPlaying) {
+    mPlaying = aPlaying;
+    UpdateStreamSuspended(mStream, !mPlaying);
+  }
+}
+
+void
+DecodedStreamData::Forget()
+{
+  mListener->Forget();
+}
+
+void
+DecodedStreamData::DumpDebugInfo()
+{
+  DUMP_LOG(
+    "DecodedStreamData=%p mPlaying=%d mAudioFramesWritten=%lld"
+    "mNextAudioTime=%lld mNextVideoTime=%lld mHaveSentFinish=%d"
+    "mHaveSentFinishAudio=%d mHaveSentFinishVideo=%d",
+    this, mPlaying, mAudioFramesWritten, mNextAudioTime, mNextVideoTime,
+    mHaveSentFinish, mHaveSentFinishAudio, mHaveSentFinishVideo);
+}
+
+DecodedStream::DecodedStream(AbstractThread* aOwnerThread,
+                             MediaQueue<MediaData>& aAudioQueue,
+                             MediaQueue<MediaData>& aVideoQueue,
+                             OutputStreamManager* aOutputStreamManager,
+                             const bool& aSameOrigin,
+                             const PrincipalHandle& aPrincipalHandle)
+  : mOwnerThread(aOwnerThread)
+  , mOutputStreamManager(aOutputStreamManager)
+  , mPlaying(false)
+  , mSameOrigin(aSameOrigin)
+  , mPrincipalHandle(aPrincipalHandle)
+  , mAudioQueue(aAudioQueue)
+  , mVideoQueue(aVideoQueue)
+{
+}
+
+DecodedStream::~DecodedStream()
+{
+  MOZ_ASSERT(mStartTime.isNothing(), "playback should've ended.");
+}
+
+const media::MediaSink::PlaybackParams&
+DecodedStream::GetPlaybackParams() const
+{
+  AssertOwnerThread();
+  return mParams;
+}
+
+void
+DecodedStream::SetPlaybackParams(const PlaybackParams& aParams)
+{
+  AssertOwnerThread();
+  mParams = aParams;
+}
+
+RefPtr<GenericPromise>
+DecodedStream::OnEnded(TrackType aType)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mStartTime.isSome());
+
+  if (aType == TrackInfo::kAudioTrack && mInfo.HasAudio()) {
+    // TODO: we should return a promise which is resolved when the audio track
+    // is finished. For now this promise is resolved when the whole stream is
+    // finished.
+    return mFinishPromise;
+  } else if (aType == TrackInfo::kVideoTrack && mInfo.HasVideo()) {
+    return mFinishPromise;
+  }
+  return nullptr;
+}
+
+void
+DecodedStream::Start(int64_t aStartTime, const MediaInfo& aInfo)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mStartTime.isNothing(), "playback already started.");
+
+  mStartTime.emplace(aStartTime);
+  mLastOutputTime = 0;
+  mInfo = aInfo;
+  mPlaying = true;
+  ConnectListener();
+
+  class R : public Runnable {
+    typedef MozPromiseHolder<GenericPromise> Promise;
+  public:
+    R(PlaybackInfoInit&& aInit, Promise&& aPromise, OutputStreamManager* aManager)
+      : mInit(Move(aInit)), mOutputStreamManager(aManager)
+    {
+      mPromise = Move(aPromise);
+    }
+    NS_IMETHOD Run() override
+    {
+      MOZ_ASSERT(NS_IsMainThread());
+      // No need to create a source stream when there are no output streams. This
+      // happens when RemoveOutput() is called immediately after StartPlayback().
+      if (!mOutputStreamManager->Graph()) {
+        // Resolve the promise to indicate the end of playback.
+        mPromise.Resolve(true, __func__);
+        return NS_OK;
+      }
+      mData = MakeUnique<DecodedStreamData>(
+        mOutputStreamManager, Move(mInit), Move(mPromise));
+      return NS_OK;
+    }
+    UniquePtr<DecodedStreamData> ReleaseData()
+    {
+      return Move(mData);
+    }
+  private:
+    PlaybackInfoInit mInit;
+    Promise mPromise;
+    RefPtr<OutputStreamManager> mOutputStreamManager;
+    UniquePtr<DecodedStreamData> mData;
+  };
+
+  MozPromiseHolder<GenericPromise> promise;
+  mFinishPromise = promise.Ensure(__func__);
+  PlaybackInfoInit init {
+    aStartTime, aInfo
+  };
+  nsCOMPtr<nsIRunnable> r = new R(Move(init), Move(promise), mOutputStreamManager);
+  nsCOMPtr<nsIThread> mainThread = do_GetMainThread();
+  SyncRunnable::DispatchToThread(mainThread, r);
+  mData = static_cast<R*>(r.get())->ReleaseData();
+
+  if (mData) {
+    mOutputListener = mData->OnOutput().Connect(
+      mOwnerThread, this, &DecodedStream::NotifyOutput);
+    mData->SetPlaying(mPlaying);
+    SendData();
+  }
+}
+
+void
+DecodedStream::Stop()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mStartTime.isSome(), "playback not started.");
+
+  mStartTime.reset();
+  DisconnectListener();
+  mFinishPromise = nullptr;
+
+  // Clear mData immediately when this playback session ends so we won't
+  // send data to the wrong stream in SendData() in next playback session.
+  DestroyData(Move(mData));
+}
+
+bool
+DecodedStream::IsStarted() const
+{
+  AssertOwnerThread();
+  return mStartTime.isSome();
+}
+
+bool
+DecodedStream::IsPlaying() const
+{
+  AssertOwnerThread();
+  return IsStarted() && mPlaying;
+}
+
+void
+DecodedStream::DestroyData(UniquePtr<DecodedStreamData> aData)
+{
+  AssertOwnerThread();
+
+  if (!aData) {
+    return;
+  }
+
+  mOutputListener.Disconnect();
+
+  DecodedStreamData* data = aData.release();
+  data->Forget();
+  nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([=] () {
+    delete data;
+  });
+  AbstractThread::MainThread()->Dispatch(r.forget());
+}
+
+void
+DecodedStream::SetPlaying(bool aPlaying)
+{
+  AssertOwnerThread();
+
+  // Resume/pause matters only when playback started.
+  if (mStartTime.isNothing()) {
+    return;
+  }
+
+  mPlaying = aPlaying;
+  if (mData) {
+    mData->SetPlaying(aPlaying);
+  }
+}
+
+void
+DecodedStream::SetVolume(double aVolume)
+{
+  AssertOwnerThread();
+  mParams.mVolume = aVolume;
+}
+
+void
+DecodedStream::SetPlaybackRate(double aPlaybackRate)
+{
+  AssertOwnerThread();
+  mParams.mPlaybackRate = aPlaybackRate;
+}
+
+void
+DecodedStream::SetPreservesPitch(bool aPreservesPitch)
+{
+  AssertOwnerThread();
+  mParams.mPreservesPitch = aPreservesPitch;
+}
+
+static void
+SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime,
+                MediaData* aData, AudioSegment* aOutput, uint32_t aRate,
+                const PrincipalHandle& aPrincipalHandle)
+{
+  // The amount of audio frames that is used to fuzz rounding errors.
+  static const int64_t AUDIO_FUZZ_FRAMES = 1;
+
+  MOZ_ASSERT(aData);
+  AudioData* audio = aData->As<AudioData>();
+  // This logic has to mimic AudioSink closely to make sure we write
+  // the exact same silences
+  CheckedInt64 audioWrittenOffset = aStream->mAudioFramesWritten +
+                                    UsecsToFrames(aStartTime, aRate);
+  CheckedInt64 frameOffset = UsecsToFrames(audio->mTime, aRate);
+
+  if (!audioWrittenOffset.isValid() ||
+      !frameOffset.isValid() ||
+      // ignore packet that we've already processed
+      audio->GetEndTime() <= aStream->mNextAudioTime) {
+    return;
+  }
+
+  if (audioWrittenOffset.value() + AUDIO_FUZZ_FRAMES < frameOffset.value()) {
+    int64_t silentFrames = frameOffset.value() - audioWrittenOffset.value();
+    // Write silence to catch up
+    AudioSegment silence;
+    silence.InsertNullDataAtStart(silentFrames);
+    aStream->mAudioFramesWritten += silentFrames;
+    audioWrittenOffset += silentFrames;
+    aOutput->AppendFrom(&silence);
+  }
+
+  // Always write the whole sample without truncation to be consistent with
+  // DecodedAudioDataSink::PlayFromAudioQueue()
+  audio->EnsureAudioBuffer();
+  RefPtr<SharedBuffer> buffer = audio->mAudioBuffer;
+  AudioDataValue* bufferData = static_cast<AudioDataValue*>(buffer->Data());
+  AutoTArray<const AudioDataValue*, 2> channels;
+  for (uint32_t i = 0; i < audio->mChannels; ++i) {
+    channels.AppendElement(bufferData + i * audio->mFrames);
+  }
+  aOutput->AppendFrames(buffer.forget(), channels, audio->mFrames, aPrincipalHandle);
+  aStream->mAudioFramesWritten += audio->mFrames;
+
+  aStream->mNextAudioTime = audio->GetEndTime();
+}
+
+void
+DecodedStream::SendAudio(double aVolume, bool aIsSameOrigin,
+                         const PrincipalHandle& aPrincipalHandle)
+{
+  AssertOwnerThread();
+
+  if (!mInfo.HasAudio()) {
+    return;
+  }
+
+  AudioSegment output;
+  uint32_t rate = mInfo.mAudio.mRate;
+  AutoTArray<RefPtr<MediaData>,10> audio;
+  TrackID audioTrackId = mInfo.mAudio.mTrackId;
+  SourceMediaStream* sourceStream = mData->mStream;
+
+  // It's OK to hold references to the AudioData because AudioData
+  // is ref-counted.
+  mAudioQueue.GetElementsAfter(mData->mNextAudioTime, &audio);
+  for (uint32_t i = 0; i < audio.Length(); ++i) {
+    SendStreamAudio(mData.get(), mStartTime.ref(), audio[i], &output, rate,
+                    aPrincipalHandle);
+  }
+
+  output.ApplyVolume(aVolume);
+
+  if (!aIsSameOrigin) {
+    output.ReplaceWithDisabled();
+  }
+
+  // |mNextAudioTime| is updated as we process each audio sample in
+  // SendStreamAudio(). This is consistent with how |mNextVideoTime|
+  // is updated for video samples.
+  if (output.GetDuration() > 0) {
+    sourceStream->AppendToTrack(audioTrackId, &output);
+  }
+
+  if (mAudioQueue.IsFinished() && !mData->mHaveSentFinishAudio) {
+    sourceStream->EndTrack(audioTrackId);
+    mData->mHaveSentFinishAudio = true;
+  }
+}
+
+static void
+WriteVideoToMediaStream(MediaStream* aStream,
+                        layers::Image* aImage,
+                        int64_t aEndMicroseconds,
+                        int64_t aStartMicroseconds,
+                        const mozilla::gfx::IntSize& aIntrinsicSize,
+                        const TimeStamp& aTimeStamp,
+                        VideoSegment* aOutput,
+                        const PrincipalHandle& aPrincipalHandle)
+{
+  RefPtr<layers::Image> image = aImage;
+  StreamTime duration =
+      aStream->MicrosecondsToStreamTimeRoundDown(aEndMicroseconds) -
+      aStream->MicrosecondsToStreamTimeRoundDown(aStartMicroseconds);
+  aOutput->AppendFrame(image.forget(), duration, aIntrinsicSize,
+                       aPrincipalHandle, false, aTimeStamp);
+}
+
+static bool
+ZeroDurationAtLastChunk(VideoSegment& aInput)
+{
+  // Get the last video frame's start time in VideoSegment aInput.
+  // If the start time is equal to the duration of aInput, means the last video
+  // frame's duration is zero.
+  StreamTime lastVideoStratTime;
+  aInput.GetLastFrame(&lastVideoStratTime);
+  return lastVideoStratTime == aInput.GetDuration();
+}
+
+void
+DecodedStream::SendVideo(bool aIsSameOrigin, const PrincipalHandle& aPrincipalHandle)
+{
+  AssertOwnerThread();
+
+  if (!mInfo.HasVideo()) {
+    return;
+  }
+
+  VideoSegment output;
+  TrackID videoTrackId = mInfo.mVideo.mTrackId;
+  AutoTArray<RefPtr<MediaData>, 10> video;
+  SourceMediaStream* sourceStream = mData->mStream;
+
+  // It's OK to hold references to the VideoData because VideoData
+  // is ref-counted.
+  mVideoQueue.GetElementsAfter(mData->mNextVideoTime, &video);
+
+  // tracksStartTimeStamp might be null when the SourceMediaStream not yet
+  // be added to MediaStreamGraph.
+  TimeStamp tracksStartTimeStamp = sourceStream->GetStreamTracksStrartTimeStamp();
+  if (tracksStartTimeStamp.IsNull()) {
+    tracksStartTimeStamp = TimeStamp::Now();
+  }
+
+  for (uint32_t i = 0; i < video.Length(); ++i) {
+    VideoData* v = video[i]->As<VideoData>();
+
+    if (mData->mNextVideoTime < v->mTime) {
+      // Write last video frame to catch up. mLastVideoImage can be null here
+      // which is fine, it just means there's no video.
+
+      // TODO: |mLastVideoImage| should come from the last image rendered
+      // by the state machine. This will avoid the black frame when capture
+      // happens in the middle of playback (especially in th middle of a
+      // video frame). E.g. if we have a video frame that is 30 sec long
+      // and capture happens at 15 sec, we'll have to append a black frame
+      // that is 15 sec long.
+      WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage, v->mTime,
+          mData->mNextVideoTime, mData->mLastVideoImageDisplaySize,
+          tracksStartTimeStamp + TimeDuration::FromMicroseconds(v->mTime),
+          &output, aPrincipalHandle);
+      mData->mNextVideoTime = v->mTime;
+    }
+
+    if (mData->mNextVideoTime < v->GetEndTime()) {
+      WriteVideoToMediaStream(sourceStream, v->mImage, v->GetEndTime(),
+          mData->mNextVideoTime, v->mDisplay,
+          tracksStartTimeStamp + TimeDuration::FromMicroseconds(v->GetEndTime()),
+          &output, aPrincipalHandle);
+      mData->mNextVideoTime = v->GetEndTime();
+      mData->mLastVideoImage = v->mImage;
+      mData->mLastVideoImageDisplaySize = v->mDisplay;
+    }
+  }
+
+  // Check the output is not empty.
+  if (output.GetLastFrame()) {
+    mData->mEOSVideoCompensation = ZeroDurationAtLastChunk(output);
+  }
+
+  if (!aIsSameOrigin) {
+    output.ReplaceWithDisabled();
+  }
+
+  if (output.GetDuration() > 0) {
+    sourceStream->AppendToTrack(videoTrackId, &output);
+  }
+
+  if (mVideoQueue.IsFinished() && !mData->mHaveSentFinishVideo) {
+    if (mData->mEOSVideoCompensation) {
+      VideoSegment endSegment;
+      // Calculate the deviation clock time from DecodedStream.
+      int64_t deviation_usec = sourceStream->StreamTimeToMicroseconds(1);
+      WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage,
+          mData->mNextVideoTime + deviation_usec, mData->mNextVideoTime,
+          mData->mLastVideoImageDisplaySize,
+          tracksStartTimeStamp + TimeDuration::FromMicroseconds(mData->mNextVideoTime + deviation_usec),
+          &endSegment, aPrincipalHandle);
+      mData->mNextVideoTime += deviation_usec;
+      MOZ_ASSERT(endSegment.GetDuration() > 0);
+      if (!aIsSameOrigin) {
+        endSegment.ReplaceWithDisabled();
+      }
+      sourceStream->AppendToTrack(videoTrackId, &endSegment);
+    }
+    sourceStream->EndTrack(videoTrackId);
+    mData->mHaveSentFinishVideo = true;
+  }
+}
+
+void
+DecodedStream::AdvanceTracks()
+{
+  AssertOwnerThread();
+
+  StreamTime endPosition = 0;
+
+  if (mInfo.HasAudio()) {
+    StreamTime audioEnd = mData->mStream->TicksToTimeRoundDown(
+        mInfo.mAudio.mRate, mData->mAudioFramesWritten);
+    endPosition = std::max(endPosition, audioEnd);
+  }
+
+  if (mInfo.HasVideo()) {
+    StreamTime videoEnd = mData->mStream->MicrosecondsToStreamTimeRoundDown(
+        mData->mNextVideoTime - mStartTime.ref());
+    endPosition = std::max(endPosition, videoEnd);
+  }
+
+  if (!mData->mHaveSentFinish) {
+    mData->mStream->AdvanceKnownTracksTime(endPosition);
+  }
+}
+
+void
+DecodedStream::SendData()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mStartTime.isSome(), "Must be called after StartPlayback()");
+
+  // Not yet created on the main thread. MDSM will try again later.
+  if (!mData) {
+    return;
+  }
+
+  // Nothing to do when the stream is finished.
+  if (mData->mHaveSentFinish) {
+    return;
+  }
+
+  SendAudio(mParams.mVolume, mSameOrigin, mPrincipalHandle);
+  SendVideo(mSameOrigin, mPrincipalHandle);
+  AdvanceTracks();
+
+  bool finished = (!mInfo.HasAudio() || mAudioQueue.IsFinished()) &&
+                  (!mInfo.HasVideo() || mVideoQueue.IsFinished());
+
+  if (finished && !mData->mHaveSentFinish) {
+    mData->mHaveSentFinish = true;
+    mData->mStream->Finish();
+  }
+}
+
+int64_t
+DecodedStream::GetEndTime(TrackType aType) const
+{
+  AssertOwnerThread();
+  if (aType == TrackInfo::kAudioTrack && mInfo.HasAudio() && mData) {
+    CheckedInt64 t = mStartTime.ref() +
+      FramesToUsecs(mData->mAudioFramesWritten, mInfo.mAudio.mRate);
+    if (t.isValid()) {
+      return t.value();
+    }
+  } else if (aType == TrackInfo::kVideoTrack && mData) {
+    return mData->mNextVideoTime;
+  }
+  return -1;
+}
+
+int64_t
+DecodedStream::GetPosition(TimeStamp* aTimeStamp) const
+{
+  AssertOwnerThread();
+  // This is only called after MDSM starts playback. So mStartTime is
+  // guaranteed to be something.
+  MOZ_ASSERT(mStartTime.isSome());
+  if (aTimeStamp) {
+    *aTimeStamp = TimeStamp::Now();
+  }
+  return mStartTime.ref() + mLastOutputTime;
+}
+
+void
+DecodedStream::NotifyOutput(int64_t aTime)
+{
+  AssertOwnerThread();
+  mLastOutputTime = aTime;
+  int64_t currentTime = GetPosition();
+
+  // Remove audio samples that have been played by MSG from the queue.
+  RefPtr<MediaData> a = mAudioQueue.PeekFront();
+  for (; a && a->mTime < currentTime;) {
+    RefPtr<MediaData> releaseMe = mAudioQueue.PopFront();
+    a = mAudioQueue.PeekFront();
+  }
+}
+
+void
+DecodedStream::ConnectListener()
+{
+  AssertOwnerThread();
+
+  mAudioPushListener = mAudioQueue.PushEvent().Connect(
+    mOwnerThread, this, &DecodedStream::SendData);
+  mAudioFinishListener = mAudioQueue.FinishEvent().Connect(
+    mOwnerThread, this, &DecodedStream::SendData);
+  mVideoPushListener = mVideoQueue.PushEvent().Connect(
+    mOwnerThread, this, &DecodedStream::SendData);
+  mVideoFinishListener = mVideoQueue.FinishEvent().Connect(
+    mOwnerThread, this, &DecodedStream::SendData);
+}
+
+void
+DecodedStream::DisconnectListener()
+{
+  AssertOwnerThread();
+
+  mAudioPushListener.Disconnect();
+  mVideoPushListener.Disconnect();
+  mAudioFinishListener.Disconnect();
+  mVideoFinishListener.Disconnect();
+}
+
+void
+DecodedStream::DumpDebugInfo()
+{
+  AssertOwnerThread();
+  DUMP_LOG(
+    "DecodedStream=%p mStartTime=%lld mLastOutputTime=%lld mPlaying=%d mData=%p",
+    this, mStartTime.valueOr(-1), mLastOutputTime, mPlaying, mData.get());
+  if (mData) {
+    mData->DumpDebugInfo();
+  }
+}
+
+} // namespace mozilla

dom/media/mediasink/DecodedStream.h

@@ -0,0 +1,122 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DecodedStream_h_
+#define DecodedStream_h_
+
+#include "MediaEventSource.h"
+#include "MediaInfo.h"
+#include "MediaSink.h"
+
+#include "mozilla/AbstractThread.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/UniquePtr.h"
+
+namespace mozilla {
+
+class DecodedStreamData;
+class MediaData;
+class MediaStream;
+class OutputStreamManager;
+struct PlaybackInfoInit;
+class ProcessedMediaStream;
+class TimeStamp;
+
+template <class T> class MediaQueue;
+
+class DecodedStream : public media::MediaSink {
+  using media::MediaSink::PlaybackParams;
+
+public:
+  DecodedStream(AbstractThread* aOwnerThread,
+                MediaQueue<MediaData>& aAudioQueue,
+                MediaQueue<MediaData>& aVideoQueue,
+                OutputStreamManager* aOutputStreamManager,
+                const bool& aSameOrigin,
+                const PrincipalHandle& aPrincipalHandle);
+
+  // MediaSink functions.
+  const PlaybackParams& GetPlaybackParams() const override;
+  void SetPlaybackParams(const PlaybackParams& aParams) override;
+
+  RefPtr<GenericPromise> OnEnded(TrackType aType) override;
+  int64_t GetEndTime(TrackType aType) const override;
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
+  bool HasUnplayedFrames(TrackType aType) const override
+  {
+    // TODO: implement this.
+    return false;
+  }
+
+  void SetVolume(double aVolume) override;
+  void SetPlaybackRate(double aPlaybackRate) override;
+  void SetPreservesPitch(bool aPreservesPitch) override;
+  void SetPlaying(bool aPlaying) override;
+
+  void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
+  void Stop() override;
+  bool IsStarted() const override;
+  bool IsPlaying() const override;
+
+  void DumpDebugInfo() override;
+
+protected:
+  virtual ~DecodedStream();
+
+private:
+  void DestroyData(UniquePtr<DecodedStreamData> aData);
+  void AdvanceTracks();
+  void SendAudio(double aVolume, bool aIsSameOrigin, const PrincipalHandle& aPrincipalHandle);
+  void SendVideo(bool aIsSameOrigin, const PrincipalHandle& aPrincipalHandle);
+  void SendData();
+  void NotifyOutput(int64_t aTime);
+
+  void AssertOwnerThread() const {
+    MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+  }
+
+  void ConnectListener();
+  void DisconnectListener();
+
+  const RefPtr<AbstractThread> mOwnerThread;
+
+  /*
+   * Main thread only members.
+   */
+  // Data about MediaStreams that are being fed by the decoder.
+  const RefPtr<OutputStreamManager> mOutputStreamManager;
+
+  /*
+   * Worker thread only members.
+   */
+  UniquePtr<DecodedStreamData> mData;
+  RefPtr<GenericPromise> mFinishPromise;
+
+  bool mPlaying;
+  const bool& mSameOrigin; // valid until Shutdown() is called.
+  const PrincipalHandle& mPrincipalHandle; // valid until Shutdown() is called.
+
+  PlaybackParams mParams;
+
+  Maybe<int64_t> mStartTime;
+  int64_t mLastOutputTime = 0; // microseconds
+  MediaInfo mInfo;
+
+  MediaQueue<MediaData>& mAudioQueue;
+  MediaQueue<MediaData>& mVideoQueue;
+
+  MediaEventListener mAudioPushListener;
+  MediaEventListener mVideoPushListener;
+  MediaEventListener mAudioFinishListener;
+  MediaEventListener mVideoFinishListener;
+  MediaEventListener mOutputListener;
+};
+
+} // namespace mozilla
+
+#endif // DecodedStream_h_

dom/media/mediasink/MediaSink.h

@@ -0,0 +1,133 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MediaSink_h_
+#define MediaSink_h_
+
+#include "mozilla/RefPtr.h"
+#include "mozilla/MozPromise.h"
+#include "nsISupportsImpl.h"
+#include "MediaInfo.h"
+
+namespace mozilla {
+
+class TimeStamp;
+
+namespace media {
+
+/**
+ * A consumer of audio/video data which plays audio and video tracks and
+ * manages A/V sync between them.
+ *
+ * A typical sink sends audio/video outputs to the speaker and screen.
+ * However, there are also sinks which capture the output of an media element
+ * and send the output to a MediaStream.
+ *
+ * This class is used to move A/V sync management and audio/video rendering
+ * out of MDSM so it is possible for subclasses to do external rendering using
+ * specific hardware which is required by TV projects and CDM.
+ *
+ * Note this class is not thread-safe and should be called from the state
+ * machine thread only.
+ */
+class MediaSink {
+public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSink);
+  typedef mozilla::TrackInfo::TrackType TrackType;
+
+  struct PlaybackParams {
+    PlaybackParams()
+      : mVolume(1.0) , mPlaybackRate(1.0) , mPreservesPitch(true) {}
+    double mVolume;
+    double mPlaybackRate;
+    bool mPreservesPitch;
+  };
+
+  // Return the playback parameters of this sink.
+  // Can be called in any state.
+  virtual const PlaybackParams& GetPlaybackParams() const = 0;
+
+  // Set the playback parameters of this sink.
+  // Can be called in any state.
+  virtual void SetPlaybackParams(const PlaybackParams& aParams) = 0;
+
+  // Return a promise which is resolved when the track finishes
+  // or null if no such track.
+  // Must be called after playback starts.
+  virtual RefPtr<GenericPromise> OnEnded(TrackType aType) = 0;
+
+  // Return the end time of the audio/video data that has been consumed
+  // or -1 if no such track.
+  // Must be called after playback starts.
+  virtual int64_t GetEndTime(TrackType aType) const = 0;
+
+  // Return playback position of the media.
+  // Since A/V sync is always maintained by this sink, there is no need to
+  // specify whether we want to get audio or video position.
+  // aTimeStamp returns the timeStamp corresponding to the returned position
+  // which is used by the compositor to derive the render time of video frames.
+  // Must be called after playback starts.
+  virtual int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const = 0;
+
+  // Return true if there are data consumed but not played yet.
+  // Can be called in any state.
+  virtual bool HasUnplayedFrames(TrackType aType) const = 0;
+
+  // Set volume of the audio track.
+  // Do nothing if this sink has no audio track.
+  // Can be called in any state.
+  virtual void SetVolume(double aVolume) {}
+
+  // Set the playback rate.
+  // Can be called in any state.
+  virtual void SetPlaybackRate(double aPlaybackRate) {}
+
+  // Whether to preserve pitch of the audio track.
+  // Do nothing if this sink has no audio track.
+  // Can be called in any state.
+  virtual void SetPreservesPitch(bool aPreservesPitch) {}
+
+  // Pause/resume the playback. Only work after playback starts.
+  virtual void SetPlaying(bool aPlaying) = 0;
+
+  // Single frame rendering operation may need to be done before playback
+  // started (1st frame) or right after seek completed or playback stopped.
+  // Do nothing if this sink has no video track. Can be called in any state.
+  virtual void Redraw(const VideoInfo& aInfo) {};
+
+  // Begin a playback session with the provided start time and media info.
+  // Must be called when playback is stopped.
+  virtual void Start(int64_t aStartTime, const MediaInfo& aInfo) = 0;
+
+  // Finish a playback session.
+  // Must be called after playback starts.
+  virtual void Stop() = 0;
+
+  // Return true if playback has started.
+  // Can be called in any state.
+  virtual bool IsStarted() const = 0;
+
+  // Return true if playback is started and not paused otherwise false.
+  // Can be called in any state.
+  virtual bool IsPlaying() const = 0;
+
+  // Called on the state machine thread to shut down the sink. All resources
+  // allocated by this sink should be released.
+  // Must be called after playback stopped.
+  virtual void Shutdown() {}
+
+  // Dump debugging information to the logs.
+  // Can be called in any phase.
+  virtual void DumpDebugInfo() {}
+
+protected:
+  virtual ~MediaSink() {}
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif //MediaSink_h_

dom/media/mediasink/OutputStreamManager.cpp

@@ -0,0 +1,134 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamGraph.h"
+#include "OutputStreamManager.h"
+
+namespace mozilla {
+
+OutputStreamData::~OutputStreamData()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  // Break the connection to the input stream if necessary.
+  if (mPort) {
+    mPort->Destroy();
+  }
+}
+
+void
+OutputStreamData::Init(OutputStreamManager* aOwner, ProcessedMediaStream* aStream)
+{
+  mOwner = aOwner;
+  mStream = aStream;
+}
+
+bool
+OutputStreamData::Connect(MediaStream* aStream)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!mPort, "Already connected?");
+
+  if (mStream->IsDestroyed()) {
+    return false;
+  }
+
+  mPort = mStream->AllocateInputPort(aStream);
+  return true;
+}
+
+bool
+OutputStreamData::Disconnect()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  // During cycle collection, DOMMediaStream can be destroyed and send
+  // its Destroy message before this decoder is destroyed. So we have to
+  // be careful not to send any messages after the Destroy().
+  if (mStream->IsDestroyed()) {
+    return false;
+  }
+
+  // Disconnect the existing port if necessary.
+  if (mPort) {
+    mPort->Destroy();
+    mPort = nullptr;
+  }
+  return true;
+}
+
+bool
+OutputStreamData::Equals(MediaStream* aStream) const
+{
+  return mStream == aStream;
+}
+
+MediaStreamGraph*
+OutputStreamData::Graph() const
+{
+  return mStream->Graph();
+}
+
+void
+OutputStreamManager::Add(ProcessedMediaStream* aStream, bool aFinishWhenEnded)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  // All streams must belong to the same graph.
+  MOZ_ASSERT(!Graph() || Graph() == aStream->Graph());
+
+  // Ensure that aStream finishes the moment mDecodedStream does.
+  if (aFinishWhenEnded) {
+    aStream->SetAutofinish(true);
+  }
+
+  OutputStreamData* p = mStreams.AppendElement();
+  p->Init(this, aStream);
+
+  // Connect to the input stream if we have one. Otherwise the output stream
+  // will be connected in Connect().
+  if (mInputStream) {
+    p->Connect(mInputStream);
+  }
+}
+
+void
+OutputStreamManager::Remove(MediaStream* aStream)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  for (int32_t i = mStreams.Length() - 1; i >= 0; --i) {
+    if (mStreams[i].Equals(aStream)) {
+      mStreams.RemoveElementAt(i);
+      break;
+    }
+  }
+}
+
+void
+OutputStreamManager::Connect(MediaStream* aStream)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  mInputStream = aStream;
+  for (int32_t i = mStreams.Length() - 1; i >= 0; --i) {
+    if (!mStreams[i].Connect(aStream)) {
+      // Probably the DOMMediaStream was GCed. Clean up.
+      mStreams.RemoveElementAt(i);
+    }
+  }
+}
+
+void
+OutputStreamManager::Disconnect()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  mInputStream = nullptr;
+  for (int32_t i = mStreams.Length() - 1; i >= 0; --i) {
+    if (!mStreams[i].Disconnect()) {
+      // Probably the DOMMediaStream was GCed. Clean up.
+      mStreams.RemoveElementAt(i);
+    }
+  }
+}
+
+} // namespace mozilla

dom/media/mediasink/OutputStreamManager.h

@@ -0,0 +1,80 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef OutputStreamManager_h
+#define OutputStreamManager_h
+
+#include "mozilla/RefPtr.h"
+#include "nsTArray.h"
+
+namespace mozilla {
+
+class MediaInputPort;
+class MediaStream;
+class MediaStreamGraph;
+class OutputStreamManager;
+class ProcessedMediaStream;
+
+class OutputStreamData {
+public:
+  ~OutputStreamData();
+  void Init(OutputStreamManager* aOwner, ProcessedMediaStream* aStream);
+
+  // Connect mStream to the input stream.
+  // Return false is mStream is already destroyed, otherwise true.
+  bool Connect(MediaStream* aStream);
+  // Disconnect mStream from its input stream.
+  // Return false is mStream is already destroyed, otherwise true.
+  bool Disconnect();
+  // Return true if aStream points to the same object as mStream.
+  // Used by OutputStreamManager to remove an output stream.
+  bool Equals(MediaStream* aStream) const;
+  // Return the graph mStream belongs to.
+  MediaStreamGraph* Graph() const;
+
+private:
+  OutputStreamManager* mOwner;
+  RefPtr<ProcessedMediaStream> mStream;
+  // mPort connects our mStream to an input stream.
+  RefPtr<MediaInputPort> mPort;
+};
+
+class OutputStreamManager {
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OutputStreamManager);
+
+public:
+  // Add the output stream to the collection.
+  void Add(ProcessedMediaStream* aStream, bool aFinishWhenEnded);
+  // Remove the output stream from the collection.
+  void Remove(MediaStream* aStream);
+  // Return true if the collection empty.
+  bool IsEmpty() const
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+    return mStreams.IsEmpty();
+  }
+  // Connect all output streams in the collection to the input stream.
+  void Connect(MediaStream* aStream);
+  // Disconnect all output streams from the input stream.
+  void Disconnect();
+  // Return the graph these streams belong to or null if empty.
+  MediaStreamGraph* Graph() const
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+    return !IsEmpty() ? mStreams[0].Graph() : nullptr;
+  }
+
+private:
+  ~OutputStreamManager() {}
+  // Keep the input stream so we can connect the output streams that
+  // are added after Connect().
+  RefPtr<MediaStream> mInputStream;
+  nsTArray<OutputStreamData> mStreams;
+};
+
+} // namespace mozilla
+
+#endif // OutputStreamManager_h

dom/media/mediasink/VideoSink.cpp

@@ -0,0 +1,486 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaQueue.h"
+#include "VideoSink.h"
+#include "MediaPrefs.h"
+
+namespace mozilla {
+
+extern LazyLogModule gMediaDecoderLog;
+
+#undef FMT
+#undef DUMP_LOG
+
+#define FMT(x, ...) "VideoSink=%p " x, this, ##__VA_ARGS__
+#define VSINK_LOG(x, ...)   MOZ_LOG(gMediaDecoderLog, LogLevel::Debug,   (FMT(x, ##__VA_ARGS__)))
+#define VSINK_LOG_V(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, (FMT(x, ##__VA_ARGS__)))
+#define DUMP_LOG(x, ...) NS_DebugBreak(NS_DEBUG_WARNING, nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, nullptr, -1)
+
+using namespace mozilla::layers;
+
+namespace media {
+
+// Minimum update frequency is 1/120th of a second, i.e. half the
+// duration of a 60-fps frame.
+static const int64_t MIN_UPDATE_INTERVAL_US = 1000000 / (60 * 2);
+
+VideoSink::VideoSink(AbstractThread* aThread,
+                     MediaSink* aAudioSink,
+                     MediaQueue<MediaData>& aVideoQueue,
+                     VideoFrameContainer* aContainer,
+                     FrameStatistics& aFrameStats,
+                     uint32_t aVQueueSentToCompositerSize)
+  : mOwnerThread(aThread)
+  , mAudioSink(aAudioSink)
+  , mVideoQueue(aVideoQueue)
+  , mContainer(aContainer)
+  , mProducerID(ImageContainer::AllocateProducerID())
+  , mFrameStats(aFrameStats)
+  , mVideoFrameEndTime(-1)
+  , mHasVideo(false)
+  , mUpdateScheduler(aThread)
+  , mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize)
+  , mMinVideoQueueSize(MediaPrefs::RuinAvSync() ? 1 : 0)
+{
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+}
+
+VideoSink::~VideoSink()
+{
+}
+
+const MediaSink::PlaybackParams&
+VideoSink::GetPlaybackParams() const
+{
+  AssertOwnerThread();
+
+  return mAudioSink->GetPlaybackParams();
+}
+
+void
+VideoSink::SetPlaybackParams(const PlaybackParams& aParams)
+{
+  AssertOwnerThread();
+
+  mAudioSink->SetPlaybackParams(aParams);
+}
+
+RefPtr<GenericPromise>
+VideoSink::OnEnded(TrackType aType)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
+
+  if (aType == TrackInfo::kAudioTrack) {
+    return mAudioSink->OnEnded(aType);
+  } else if (aType == TrackInfo::kVideoTrack) {
+    return mEndPromise;
+  }
+  return nullptr;
+}
+
+int64_t
+VideoSink::GetEndTime(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
+
+  if (aType == TrackInfo::kVideoTrack) {
+    return mVideoFrameEndTime;
+  } else if (aType == TrackInfo::kAudioTrack) {
+    return mAudioSink->GetEndTime(aType);
+  }
+  return -1;
+}
+
+int64_t
+VideoSink::GetPosition(TimeStamp* aTimeStamp) const
+{
+  AssertOwnerThread();
+
+  return mAudioSink->GetPosition(aTimeStamp);
+}
+
+bool
+VideoSink::HasUnplayedFrames(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(aType == TrackInfo::kAudioTrack, "Not implemented for non audio tracks.");
+
+  return mAudioSink->HasUnplayedFrames(aType);
+}
+
+void
+VideoSink::SetPlaybackRate(double aPlaybackRate)
+{
+  AssertOwnerThread();
+
+  mAudioSink->SetPlaybackRate(aPlaybackRate);
+}
+
+void
+VideoSink::SetVolume(double aVolume)
+{
+  AssertOwnerThread();
+
+  mAudioSink->SetVolume(aVolume);
+}
+
+void
+VideoSink::SetPreservesPitch(bool aPreservesPitch)
+{
+  AssertOwnerThread();
+
+  mAudioSink->SetPreservesPitch(aPreservesPitch);
+}
+
+void
+VideoSink::SetPlaying(bool aPlaying)
+{
+  AssertOwnerThread();
+  VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying);
+
+  if (!aPlaying) {
+    // Reset any update timer if paused.
+    mUpdateScheduler.Reset();
+    // Since playback is paused, tell compositor to render only current frame.
+    RenderVideoFrames(1);
+    if (mContainer) {
+      mContainer->ClearCachedResources();
+    }
+  }
+
+  mAudioSink->SetPlaying(aPlaying);
+
+  if (mHasVideo && aPlaying) {
+    // There's no thread in VideoSink for pulling video frames, need to trigger
+    // rendering while becoming playing status. because the VideoQueue may be
+    // full already.
+    TryUpdateRenderedVideoFrames();
+  }
+}
+
+void
+VideoSink::Start(int64_t aStartTime, const MediaInfo& aInfo)
+{
+  AssertOwnerThread();
+  VSINK_LOG("[%s]", __func__);
+
+  mAudioSink->Start(aStartTime, aInfo);
+
+  mHasVideo = aInfo.HasVideo();
+
+  if (mHasVideo) {
+    mEndPromise = mEndPromiseHolder.Ensure(__func__);
+
+    // If the underlying MediaSink has an end promise for the video track (which
+    // happens when mAudioSink refers to a DecodedStream), we must wait for it
+    // to complete before resolving our own end promise. Otherwise, MDSM might
+    // stop playback before DecodedStream plays to the end and cause
+    // test_streams_element_capture.html to time out.
+    RefPtr<GenericPromise> p = mAudioSink->OnEnded(TrackInfo::kVideoTrack);
+    if (p) {
+      RefPtr<VideoSink> self = this;
+      mVideoSinkEndRequest.Begin(p->Then(mOwnerThread, __func__,
+        [self] () {
+          self->mVideoSinkEndRequest.Complete();
+          self->TryUpdateRenderedVideoFrames();
+          // It is possible the video queue size is 0 and we have no frames to
+          // render. However, we need to call MaybeResolveEndPromise() to ensure
+          // mEndPromiseHolder is resolved.
+          self->MaybeResolveEndPromise();
+        }, [self] () {
+          self->mVideoSinkEndRequest.Complete();
+          self->TryUpdateRenderedVideoFrames();
+          self->MaybeResolveEndPromise();
+        }));
+    }
+
+    ConnectListener();
+    // Run the render loop at least once so we can resolve the end promise
+    // when video duration is 0.
+    UpdateRenderedVideoFrames();
+  }
+}
+
+void
+VideoSink::Stop()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started.");
+  VSINK_LOG("[%s]", __func__);
+
+  mAudioSink->Stop();
+
+  mUpdateScheduler.Reset();
+  if (mHasVideo) {
+    DisconnectListener();
+    mVideoSinkEndRequest.DisconnectIfExists();
+    mEndPromiseHolder.ResolveIfExists(true, __func__);
+    mEndPromise = nullptr;
+  }
+  mVideoFrameEndTime = -1;
+}
+
+bool
+VideoSink::IsStarted() const
+{
+  AssertOwnerThread();
+
+  return mAudioSink->IsStarted();
+}
+
+bool
+VideoSink::IsPlaying() const
+{
+  AssertOwnerThread();
+
+  return mAudioSink->IsPlaying();
+}
+
+void
+VideoSink::Shutdown()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");
+  VSINK_LOG("[%s]", __func__);
+
+  mAudioSink->Shutdown();
+}
+
+void
+VideoSink::OnVideoQueuePushed(RefPtr<MediaData>&& aSample)
+{
+  AssertOwnerThread();
+  // Listen to push event, VideoSink should try rendering ASAP if first frame
+  // arrives but update scheduler is not triggered yet.
+  VideoData* v = aSample->As<VideoData>();
+  if (!v->mSentToCompositor) {
+    // Since we push rendered frames back to the queue, we will receive
+    // push events for them. We only need to trigger render loop
+    // when this frame is not rendered yet.
+    TryUpdateRenderedVideoFrames();
+  }
+}
+
+void
+VideoSink::OnVideoQueueFinished()
+{
+  AssertOwnerThread();
+  // Run render loop if the end promise is not resolved yet.
+  if (!mUpdateScheduler.IsScheduled() &&
+      mAudioSink->IsPlaying() &&
+      !mEndPromiseHolder.IsEmpty()) {
+    UpdateRenderedVideoFrames();
+  }
+}
+
+void
+VideoSink::Redraw(const VideoInfo& aInfo)
+{
+  AssertOwnerThread();
+
+  // No video track, nothing to draw.
+  if (!aInfo.IsValid() || !mContainer) {
+    return;
+  }
+
+  if (VideoQueue().GetSize() > 0) {
+    RenderVideoFrames(1);
+    return;
+  }
+
+  // When we reach here, it means there are no frames in this video track.
+  // Draw a blank frame to ensure there is something in the image container
+  // to fire 'loadeddata'.
+  RefPtr<Image> blank =
+    mContainer->GetImageContainer()->CreatePlanarYCbCrImage();
+  mContainer->SetCurrentFrame(aInfo.mDisplay, blank, TimeStamp::Now());
+}
+
+void
+VideoSink::TryUpdateRenderedVideoFrames()
+{
+  AssertOwnerThread();
+  if (!mUpdateScheduler.IsScheduled() && VideoQueue().GetSize() >= 1 &&
+      mAudioSink->IsPlaying()) {
+    UpdateRenderedVideoFrames();
+  }
+}
+
+void
+VideoSink::UpdateRenderedVideoFramesByTimer()
+{
+  AssertOwnerThread();
+  mUpdateScheduler.CompleteRequest();
+  UpdateRenderedVideoFrames();
+}
+
+void
+VideoSink::ConnectListener()
+{
+  AssertOwnerThread();
+  mPushListener = VideoQueue().PushEvent().Connect(
+    mOwnerThread, this, &VideoSink::OnVideoQueuePushed);
+  mFinishListener = VideoQueue().FinishEvent().Connect(
+    mOwnerThread, this, &VideoSink::OnVideoQueueFinished);
+}
+
+void
+VideoSink::DisconnectListener()
+{
+  AssertOwnerThread();
+  mPushListener.Disconnect();
+  mFinishListener.Disconnect();
+}
+
+void
+VideoSink::RenderVideoFrames(int32_t aMaxFrames,
+                             int64_t aClockTime,
+                             const TimeStamp& aClockTimeStamp)
+{
+  AssertOwnerThread();
+
+  AutoTArray<RefPtr<MediaData>,16> frames;
+  VideoQueue().GetFirstElements(aMaxFrames, &frames);
+  if (frames.IsEmpty() || !mContainer) {
+    return;
+  }
+
+  AutoTArray<ImageContainer::NonOwningImage,16> images;
+  TimeStamp lastFrameTime;
+  MediaSink::PlaybackParams params = mAudioSink->GetPlaybackParams();
+  for (uint32_t i = 0; i < frames.Length(); ++i) {
+    VideoData* frame = frames[i]->As<VideoData>();
+
+    frame->mSentToCompositor = true;
+
+    if (!frame->mImage || !frame->mImage->IsValid() ||
+        !frame->mImage->GetSize().width || !frame->mImage->GetSize().height) {
+      continue;
+    }
+
+    int64_t frameTime = frame->mTime;
+    if (frameTime < 0) {
+      // Frame times before the start time are invalid; drop such frames
+      continue;
+    }
+
+    TimeStamp t;
+    if (aMaxFrames > 1) {
+      MOZ_ASSERT(!aClockTimeStamp.IsNull());
+      int64_t delta = frame->mTime - aClockTime;
+      t = aClockTimeStamp +
+          TimeDuration::FromMicroseconds(delta / params.mPlaybackRate);
+      if (!lastFrameTime.IsNull() && t <= lastFrameTime) {
+        // Timestamps out of order; drop the new frame. In theory we should
+        // probably replace the previous frame with the new frame if the
+        // timestamps are equal, but this is a corrupt video file already so
+        // never mind.
+        continue;
+      }
+      lastFrameTime = t;
+    }
+
+    ImageContainer::NonOwningImage* img = images.AppendElement();
+    img->mTimeStamp = t;
+    img->mImage = frame->mImage;
+    img->mFrameID = frame->mFrameID;
+    img->mProducerID = mProducerID;
+
+    VSINK_LOG_V("playing video frame %lld (id=%x) (vq-queued=%i)",
+                frame->mTime, frame->mFrameID, VideoQueue().GetSize());
+  }
+  mContainer->SetCurrentFrames(frames[0]->As<VideoData>()->mDisplay, images);
+}
+
+void
+VideoSink::UpdateRenderedVideoFrames()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink->IsPlaying(), "should be called while playing.");
+
+  // Get the current playback position.
+  TimeStamp nowTime;
+  const int64_t clockTime = mAudioSink->GetPosition(&nowTime);
+  NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
+
+  // Skip frames up to the playback position.
+  int64_t lastFrameEndTime = 0;
+  while (VideoQueue().GetSize() > mMinVideoQueueSize &&
+         clockTime >= VideoQueue().PeekFront()->GetEndTime()) {
+    RefPtr<MediaData> frame = VideoQueue().PopFront();
+    lastFrameEndTime = frame->GetEndTime();
+    if (frame->As<VideoData>()->mSentToCompositor) {
+      mFrameStats.NotifyPresentedFrame();
+    } else {
+      mFrameStats.NotifyDecodedFrames({ 0, 0, 1 });
+      VSINK_LOG_V("discarding video frame mTime=%lld clock_time=%lld",
+                  frame->mTime, clockTime);
+    }
+  }
+
+  // The presentation end time of the last video frame displayed is either
+  // the end time of the current frame, or if we dropped all frames in the
+  // queue, the end time of the last frame we removed from the queue.
+  RefPtr<MediaData> currentFrame = VideoQueue().PeekFront();
+  mVideoFrameEndTime = std::max(mVideoFrameEndTime,
+    currentFrame ? currentFrame->GetEndTime() : lastFrameEndTime);
+
+  MaybeResolveEndPromise();
+
+  RenderVideoFrames(mVideoQueueSendToCompositorSize, clockTime, nowTime);
+
+  // Get the timestamp of the next frame. Schedule the next update at
+  // the start time of the next frame. If we don't have a next frame,
+  // we will run render loops again upon incoming frames.
+  nsTArray<RefPtr<MediaData>> frames;
+  VideoQueue().GetFirstElements(2, &frames);
+  if (frames.Length() < 2) {
+    return;
+  }
+
+  int64_t nextFrameTime = frames[1]->mTime;
+  int64_t delta = std::max<int64_t>((nextFrameTime - clockTime), MIN_UPDATE_INTERVAL_US);
+  TimeStamp target = nowTime + TimeDuration::FromMicroseconds(
+     delta / mAudioSink->GetPlaybackParams().mPlaybackRate);
+
+  RefPtr<VideoSink> self = this;
+  mUpdateScheduler.Ensure(target, [self] () {
+    self->UpdateRenderedVideoFramesByTimer();
+  }, [self] () {
+    self->UpdateRenderedVideoFramesByTimer();
+  });
+}
+
+void
+VideoSink::MaybeResolveEndPromise()
+{
+  AssertOwnerThread();
+  // All frames are rendered, Let's resolve the promise.
+  if (VideoQueue().IsFinished() &&
+      VideoQueue().GetSize() <= 1 &&
+      !mVideoSinkEndRequest.Exists()) {
+    mEndPromiseHolder.ResolveIfExists(true, __func__);
+  }
+}
+
+void
+VideoSink::DumpDebugInfo()
+{
+  AssertOwnerThread();
+  DUMP_LOG(
+    "IsStarted=%d IsPlaying=%d, VideoQueue: finished=%d size=%d, "
+    "mVideoFrameEndTime=%lld mHasVideo=%d mVideoSinkEndRequest.Exists()=%d "
+    "mEndPromiseHolder.IsEmpty()=%d",
+    IsStarted(), IsPlaying(), VideoQueue().IsFinished(), VideoQueue().GetSize(),
+    mVideoFrameEndTime, mHasVideo, mVideoSinkEndRequest.Exists(), mEndPromiseHolder.IsEmpty());
+  mAudioSink->DumpDebugInfo();
+}
+
+} // namespace media
+} // namespace mozilla

dom/media/mediasink/VideoSink.h

@@ -0,0 +1,160 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef VideoSink_h_
+#define VideoSink_h_
+
+#include "FrameStatistics.h"
+#include "ImageContainer.h"
+#include "MediaEventSource.h"
+#include "MediaSink.h"
+#include "MediaTimer.h"
+#include "mozilla/AbstractThread.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/TimeStamp.h"
+#include "VideoFrameContainer.h"
+
+namespace mozilla {
+
+class VideoFrameContainer;
+template <class T> class MediaQueue;
+
+namespace media {
+
+class VideoSink : public MediaSink
+{
+  typedef mozilla::layers::ImageContainer::ProducerID ProducerID;
+public:
+  VideoSink(AbstractThread* aThread,
+            MediaSink* aAudioSink,
+            MediaQueue<MediaData>& aVideoQueue,
+            VideoFrameContainer* aContainer,
+            FrameStatistics& aFrameStats,
+            uint32_t aVQueueSentToCompositerSize);
+
+  const PlaybackParams& GetPlaybackParams() const override;
+
+  void SetPlaybackParams(const PlaybackParams& aParams) override;
+
+  RefPtr<GenericPromise> OnEnded(TrackType aType) override;
+
+  int64_t GetEndTime(TrackType aType) const override;
+
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
+
+  bool HasUnplayedFrames(TrackType aType) const override;
+
+  void SetPlaybackRate(double aPlaybackRate) override;
+
+  void SetVolume(double aVolume) override;
+
+  void SetPreservesPitch(bool aPreservesPitch) override;
+
+  void SetPlaying(bool aPlaying) override;
+
+  void Redraw(const VideoInfo& aInfo) override;
+
+  void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
+
+  void Stop() override;
+
+  bool IsStarted() const override;
+
+  bool IsPlaying() const override;
+
+  void Shutdown() override;
+
+  void DumpDebugInfo() override;
+
+private:
+  virtual ~VideoSink();
+
+  // VideoQueue listener related.
+  void OnVideoQueuePushed(RefPtr<MediaData>&& aSample);
+  void OnVideoQueueFinished();
+  void ConnectListener();
+  void DisconnectListener();
+
+  // Sets VideoQueue images into the VideoFrameContainer. Called on the shared
+  // state machine thread. The first aMaxFrames (at most) are set.
+  // aClockTime and aClockTimeStamp are used as the baseline for deriving
+  // timestamps for the frames; when omitted, aMaxFrames must be 1 and
+  // a null timestamp is passed to the VideoFrameContainer.
+  // If the VideoQueue is empty, this does nothing.
+  void RenderVideoFrames(int32_t aMaxFrames, int64_t aClockTime = 0,
+                         const TimeStamp& aClickTimeStamp = TimeStamp());
+
+  // Triggered while videosink is started, videosink becomes "playing" status,
+  // or VideoQueue event arrived.
+  void TryUpdateRenderedVideoFrames();
+
+  // If we have video, display a video frame if it's time for display has
+  // arrived, otherwise sleep until it's time for the next frame. Update the
+  // current frame time as appropriate, and trigger ready state update.
+  // Called on the shared state machine thread.
+  void UpdateRenderedVideoFrames();
+  void UpdateRenderedVideoFramesByTimer();
+
+  void MaybeResolveEndPromise();
+
+  void AssertOwnerThread() const
+  {
+    MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+  }
+
+  MediaQueue<MediaData>& VideoQueue() const {
+    return mVideoQueue;
+  }
+
+  const RefPtr<AbstractThread> mOwnerThread;
+  RefPtr<MediaSink> mAudioSink;
+  MediaQueue<MediaData>& mVideoQueue;
+  VideoFrameContainer* mContainer;
+
+  // Producer ID to help ImageContainer distinguish different streams of
+  // FrameIDs. A unique and immutable value per VideoSink.
+  const ProducerID mProducerID;
+
+  // Used to notify MediaDecoder's frame statistics
+  FrameStatistics& mFrameStats;
+
+  RefPtr<GenericPromise> mEndPromise;
+  MozPromiseHolder<GenericPromise> mEndPromiseHolder;
+  MozPromiseRequestHolder<GenericPromise> mVideoSinkEndRequest;
+
+  // The presentation end time of the last video frame which has been displayed
+  // in microseconds.
+  int64_t mVideoFrameEndTime;
+
+  // Event listeners for VideoQueue
+  MediaEventListener mPushListener;
+  MediaEventListener mFinishListener;
+
+  // True if this sink is going to handle video track.
+  bool mHasVideo;
+
+  // Used to trigger another update of rendered frames in next round.
+  DelayedScheduler mUpdateScheduler;
+
+  // Max frame number sent to compositor at a time.
+  // Based on the pref value obtained in MDSM.
+  const uint32_t mVideoQueueSendToCompositorSize;
+
+  // Talos tests for the compositor require at least one frame in the
+  // video queue so that the compositor has something to composit during
+  // the talos test when the decode is stressed. We have a minimum size
+  // on the video queue in order to facilitate this talos test.
+  // Note: Normal playback should not have a queue size of more than 0,
+  // otherwise A/V sync will be ruined! *Only* make this non-zero for
+  // testing purposes.
+  const uint32_t mMinVideoQueueSize;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif

dom/media/mediasink/moz.build

@@ -0,0 +1,18 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+UNIFIED_SOURCES += [
+    'AudioSinkWrapper.cpp',
+    'DecodedAudioDataSink.cpp',
+    'DecodedStream.cpp',
+    'OutputStreamManager.cpp',
+    'VideoSink.cpp',
+]
+
+FINAL_LIBRARY = 'xul'
+
+if CONFIG['GNU_CXX']:
+    CXXFLAGS += ['-Wno-error=shadow']