wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-27 19:07:31 +09:00
Code Issues Projects Releases Packages Wiki Activity

[WebRTC] updated to upstream branch 49 and related.

Browse source 
this patch is based on 38d7a7883...226f2f0bd
This commit is contained in:
roytam1 2021-10-01 10:48:39 +08:00
commit a3239eaa4d
2838 changed files with 151221 additions and 159688 deletions
Download patch file Download diff file Expand all files Collapse all files

33
dom/media/systemservices/CamerasChild.cpp
View file

@ -5,7 +5,6 @@
#include "CamerasChild.h"
#include "webrtc/video_engine/include/vie_capture.h"
#undef FF
#include "mozilla/Assertions.h"
@ -336,7 +335,7 @@ int
CamerasChild::GetCaptureCapability(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int capability_number,
webrtc::CaptureCapability& capability)
webrtc::VideoCaptureCapability& capability)
{
LOG(("GetCaptureCapability: %s %d", unique_idUTF8, capability_number));
RefPtr<CamerasChild> deathGrip = this;
@ -352,7 +351,7 @@ CamerasChild::GetCaptureCapability(CaptureEngine aCapEngine,
}
bool
CamerasChild::RecvReplyGetCaptureCapability(const CaptureCapability& ipcCapability)
CamerasChild::RecvReplyGetCaptureCapability(const VideoCaptureCapability& ipcCapability)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
@ -414,7 +413,7 @@ int
CamerasChild::AllocateCaptureDevice(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int unique_idUTF8Length,
int& capture_id,
int& aStreamId,
const nsACString& aOrigin)
{
LOG((__PRETTY_FUNCTION__));
@ -427,7 +426,7 @@ CamerasChild::AllocateCaptureDevice(CaptureEngine aCapEngine,
LockAndDispatch<> dispatcher(this, __func__, runnable);
if (dispatcher.Success()) {
LOG(("Capture Device allocated: %d", mReplyInteger));
capture_id = mReplyInteger;
aStreamId = mReplyInteger;
}
return dispatcher.ReturnValue();
}
@ -460,7 +459,7 @@ CamerasChild::ReleaseCaptureDevice(CaptureEngine aCapEngine,
void
CamerasChild::AddCallback(const CaptureEngine aCapEngine, const int capture_id,
webrtc::ExternalRenderer* render)
FrameRelay* render)
{
MutexAutoLock lock(mCallbackMutex);
CapturerElement ce;
@ -486,12 +485,12 @@ CamerasChild::RemoveCallback(const CaptureEngine aCapEngine, const int capture_i
int
CamerasChild::StartCapture(CaptureEngine aCapEngine,
const int capture_id,
webrtc::CaptureCapability& webrtcCaps,
webrtc::ExternalRenderer* cb)
webrtc::VideoCaptureCapability& webrtcCaps,
FrameRelay* cb)
{
LOG((__PRETTY_FUNCTION__));
AddCallback(aCapEngine, capture_id, cb);
CaptureCapability capCap(webrtcCaps.width,
VideoCaptureCapability capCap(webrtcCaps.width,
webrtcCaps.height,
webrtcCaps.maxFPS,
webrtcCaps.expectedCaptureDelay,
@ -500,7 +499,7 @@ CamerasChild::StartCapture(CaptureEngine aCapEngine,
webrtcCaps.interlaced);
RefPtr<CamerasChild> deathGrip = this;
nsCOMPtr<nsIRunnable> runnable =
mozilla::NewNonOwningRunnableMethod<CaptureEngine, int, CaptureCapability>
mozilla::NewNonOwningRunnableMethod<CaptureEngine, int, VideoCaptureCapability>
(this, &CamerasChild::SendStartCapture, aCapEngine, capture_id, capCap);
LockAndDispatch<> dispatcher(this, __func__, runnable);
return dispatcher.ReturnValue();
@ -617,18 +616,12 @@ bool
CamerasChild::RecvDeliverFrame(const CaptureEngine& capEngine,
const int& capId,
mozilla::ipc::Shmem&& shmem,
const size_t& size,
const uint32_t& time_stamp,
const int64_t& ntp_time,
const int64_t& render_time)
const VideoFrameProperties & prop)
{
MutexAutoLock lock(mCallbackMutex);
if (Callback(capEngine, capId)) {
unsigned char* image = shmem.get<unsigned char>();
Callback(capEngine, capId)->DeliverFrame(image, size,
time_stamp,
ntp_time, render_time,
nullptr);
Callback(capEngine, capId)->DeliverFrame(image, prop);
} else {
LOG(("DeliverFrame called with dead callback"));
}
@ -673,7 +666,7 @@ CamerasChild::RecvFrameSizeChange(const CaptureEngine& capEngine,
LOG((__PRETTY_FUNCTION__));
MutexAutoLock lock(mCallbackMutex);
if (Callback(capEngine, capId)) {
Callback(capEngine, capId)->FrameSizeChange(w, h, 0);
Callback(capEngine, capId)->FrameSizeChange(w, h);
} else {
LOG(("Frame size change with dead callback"));
}
@ -717,7 +710,7 @@ CamerasChild::~CamerasChild()
MOZ_COUNT_DTOR(CamerasChild);
}
webrtc::ExternalRenderer* CamerasChild::Callback(CaptureEngine aCapEngine,
FrameRelay* CamerasChild::Callback(CaptureEngine aCapEngine,
int capture_id)
{
for (unsigned int i = 0; i < mCallbacks.Length(); i++) {

37
dom/media/systemservices/CamerasChild.h
View file

@ -19,10 +19,10 @@
// conflicts with #include of scoped_ptr.h
#undef FF
#include "webrtc/common.h"
// Video Engine
#include "webrtc/video_engine/include/vie_base.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "webrtc/video_engine/include/vie_render.h"
#include "webrtc/video_renderer.h"
#include "webrtc/modules/video_capture/video_capture_defines.h"
namespace mozilla {
@ -32,10 +32,17 @@ class BackgroundChildImpl;
namespace camera {
class FrameRelay {
public:
virtual int DeliverFrame(uint8_t* buffer,
const mozilla::camera::VideoFrameProperties& props) = 0;
virtual void FrameSizeChange(unsigned int w, unsigned int h) = 0;
};
struct CapturerElement {
CaptureEngine engine;
int id;
webrtc::ExternalRenderer* callback;
FrameRelay* callback;
};
// Forward declaration so we can work with pointers to it.
@ -157,9 +164,9 @@ public:
// IPC messages recevied, received on the PBackground thread
// these are the actual callbacks with data
virtual bool RecvDeliverFrame(const CaptureEngine&, const int&, mozilla::ipc::Shmem&&,
const size_t&, const uint32_t&, const int64_t&,
const int64_t&) override;
virtual bool RecvDeliverFrame(const CaptureEngine&, const int&,
mozilla::ipc::Shmem&&,
const VideoFrameProperties & prop) override;
virtual bool RecvFrameSizeChange(const CaptureEngine&, const int&,
const int& w, const int& h) override;
@ -171,7 +178,7 @@ public:
virtual bool RecvReplyNumberOfCaptureDevices(const int&) override;
virtual bool RecvReplyNumberOfCapabilities(const int&) override;
virtual bool RecvReplyAllocateCaptureDevice(const int&) override;
virtual bool RecvReplyGetCaptureCapability(const CaptureCapability& capability) override;
virtual bool RecvReplyGetCaptureCapability(const VideoCaptureCapability& capability) override;
virtual bool RecvReplyGetCaptureDevice(const nsCString& device_name,
const nsCString& device_id,
const bool& scary) override;
@ -188,8 +195,8 @@ public:
int ReleaseCaptureDevice(CaptureEngine aCapEngine,
const int capture_id);
int StartCapture(CaptureEngine aCapEngine,
const int capture_id, webrtc::CaptureCapability& capability,
webrtc::ExternalRenderer* func);
const int capture_id, webrtc::VideoCaptureCapability& capability,
FrameRelay* func);
int StopCapture(CaptureEngine aCapEngine, const int capture_id);
int AllocateCaptureDevice(CaptureEngine aCapEngine,
const char* unique_idUTF8,
@ -199,7 +206,7 @@ public:
int GetCaptureCapability(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int capability_number,
webrtc::CaptureCapability& capability);
webrtc::VideoCaptureCapability& capability);
int GetCaptureDevice(CaptureEngine aCapEngine,
unsigned int list_number, char* device_nameUTF8,
const unsigned int device_nameUTF8Length,
@ -209,7 +216,7 @@ public:
void ShutdownAll();
int EnsureInitialized(CaptureEngine aCapEngine);
webrtc::ExternalRenderer* Callback(CaptureEngine aCapEngine, int capture_id);
FrameRelay* Callback(CaptureEngine aCapEngine, int capture_id);
private:
CamerasChild();
@ -219,7 +226,7 @@ private:
bool DispatchToParent(nsIRunnable* aRunnable,
MonitorAutoLock& aMonitor);
void AddCallback(const CaptureEngine aCapEngine, const int capture_id,
webrtc::ExternalRenderer* render);
FrameRelay* render);
void RemoveCallback(const CaptureEngine aCapEngine, const int capture_id);
void ShutdownParent();
void ShutdownChild();
@ -246,7 +253,7 @@ private:
// Async responses data contents;
bool mReplySuccess;
int mReplyInteger;
webrtc::CaptureCapability mReplyCapability;
webrtc::VideoCaptureCapability mReplyCapability;
nsCString mReplyDeviceName;
nsCString mReplyDeviceID;
bool mReplyScary;

421
dom/media/systemservices/CamerasParent.cpp
View file

@ -6,6 +6,7 @@
#include "CamerasParent.h"
#include "MediaEngine.h"
#include "MediaUtils.h"
#include "VideoFrameUtils.h"
#include "mozilla/Assertions.h"
#include "mozilla/Unused.h"
@ -26,9 +27,11 @@
#endif
#undef LOG
#undef LOG_VERBOSE
#undef LOG_ENABLED
mozilla::LazyLogModule gCamerasParentLog("CamerasParent");
#define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args)
#define LOG_VERBOSE(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Verbose, args)
#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug)
namespace mozilla {
@ -44,7 +47,7 @@ namespace camera {
// suitable for UI access.
// InputObserver is owned by CamerasParent, and it has a ref to CamerasParent
void InputObserver::DeviceChange() {
void InputObserver::OnDeviceChange() {
LOG((__PRETTY_FUNCTION__));
MOZ_ASSERT(mParent);
@ -63,80 +66,30 @@ void InputObserver::DeviceChange() {
thread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
};
class FrameSizeChangeRunnable : public Runnable {
class DeliverFrameRunnable : public ::mozilla::Runnable {
public:
FrameSizeChangeRunnable(CamerasParent *aParent, CaptureEngine capEngine,
int cap_id, unsigned int aWidth, unsigned int aHeight)
: mParent(aParent), mCapEngine(capEngine), mCapId(cap_id),
mWidth(aWidth), mHeight(aHeight) {}
NS_IMETHOD Run() override {
if (mParent->IsShuttingDown()) {
// Communication channel is being torn down
LOG(("FrameSizeChangeRunnable is active without active Child"));
mResult = 0;
return NS_OK;
}
if (!mParent->SendFrameSizeChange(mCapEngine, mCapId, mWidth, mHeight)) {
mResult = -1;
} else {
mResult = 0;
}
return NS_OK;
}
int GetResult() {
return mResult;
}
private:
RefPtr<CamerasParent> mParent;
CaptureEngine mCapEngine;
int mCapId;
unsigned int mWidth;
unsigned int mHeight;
int mResult;
};
int
CallbackHelper::FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams)
{
LOG(("CallbackHelper Video FrameSizeChange: %ux%u", w, h));
RefPtr<FrameSizeChangeRunnable> runnable =
new FrameSizeChangeRunnable(mParent, mCapEngine, mCapturerId, w, h);
MOZ_ASSERT(mParent);
nsIThread * thread = mParent->GetBackgroundThread();
MOZ_ASSERT(thread != nullptr);
thread->Dispatch(runnable, NS_DISPATCH_NORMAL);
return 0;
}
class DeliverFrameRunnable : public Runnable {
public:
DeliverFrameRunnable(CamerasParent *aParent,
CaptureEngine engine,
int cap_id,
ShmemBuffer buffer,
unsigned char* altbuffer,
size_t size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time)
: mParent(aParent), mCapEngine(engine), mCapId(cap_id), mBuffer(Move(buffer)),
mSize(size), mTimeStamp(time_stamp), mNtpTime(ntp_time),
mRenderTime(render_time) {
DeliverFrameRunnable(CamerasParent *aParent, CaptureEngine aEngine,
uint32_t aStreamId, const webrtc::VideoFrame& aFrame,
const VideoFrameProperties& aProperties)
: mParent(aParent), mCapEngine(aEngine), mStreamId(aStreamId),
mProperties(aProperties)
{
// No ShmemBuffer (of the right size) was available, so make an
// extra buffer here. We have no idea when we are going to run and
// it will be potentially long after the webrtc frame callback has
// returned, so the copy needs to be no later than here.
// We will need to copy this back into a Shmem later on so we prefer
// using ShmemBuffers to avoid the extra copy.
if (altbuffer != nullptr) {
mAlternateBuffer.reset(new unsigned char[size]);
memcpy(mAlternateBuffer.get(), altbuffer, size);
}
};
mAlternateBuffer.reset(new unsigned char[aProperties.bufferSize()]);
VideoFrameUtils::CopyVideoFrameBuffers(mAlternateBuffer.get(),
aProperties.bufferSize(), aFrame);
}
DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine,
uint32_t aStreamId, ShmemBuffer aBuffer, VideoFrameProperties& aProperties)
: mParent(aParent), mCapEngine(aEngine), mStreamId(aStreamId),
mBuffer(Move(aBuffer)), mProperties(aProperties)
{};
NS_IMETHOD Run() override {
if (mParent->IsShuttingDown()) {
@ -144,10 +97,8 @@ public:
mResult = 0;
return NS_OK;
}
if (!mParent->DeliverFrameOverIPC(mCapEngine, mCapId,
Move(mBuffer), mAlternateBuffer.get(),
mSize, mTimeStamp,
mNtpTime, mRenderTime)) {
if (!mParent->DeliverFrameOverIPC(mCapEngine, mStreamId, Move(mBuffer),
mAlternateBuffer.get(), mProperties)) {
mResult = -1;
} else {
mResult = 0;
@ -162,13 +113,10 @@ public:
private:
RefPtr<CamerasParent> mParent;
CaptureEngine mCapEngine;
int mCapId;
uint32_t mStreamId;
ShmemBuffer mBuffer;
mozilla::UniquePtr<unsigned char[]> mAlternateBuffer;
size_t mSize;
uint32_t mTimeStamp;
int64_t mNtpTime;
int64_t mRenderTime;
VideoFrameProperties mProperties;
int mResult;
};
@ -258,14 +206,11 @@ CamerasParent::StopVideoCapture()
}
int
CamerasParent::DeliverFrameOverIPC(CaptureEngine cap_engine,
int cap_id,
ShmemBuffer buffer,
unsigned char* altbuffer,
size_t size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time)
CamerasParent::DeliverFrameOverIPC(CaptureEngine capEng,
uint32_t aStreamId,
ShmemBuffer buffer,
unsigned char* altbuffer,
VideoFrameProperties& aProps)
{
// No ShmemBuffers were available, so construct one now of the right size
// and copy into it. That is an extra copy, but we expect this to be
@ -273,7 +218,7 @@ CamerasParent::DeliverFrameOverIPC(CaptureEngine cap_engine,
// buffer of the right size.
if (altbuffer != nullptr) {
// Get a shared memory buffer from the pool, at least size big
ShmemBuffer shMemBuff = mShmemPool.Get(this, size);
ShmemBuffer shMemBuff = mShmemPool.Get(this, aProps.bufferSize());
if (!shMemBuff.Valid()) {
LOG(("No usable Video shmem in DeliverFrame (out of buffers?)"));
@ -282,20 +227,18 @@ CamerasParent::DeliverFrameOverIPC(CaptureEngine cap_engine,
}
// get() and Size() check for proper alignment of the segment
memcpy(shMemBuff.GetBytes(), altbuffer, size);
memcpy(shMemBuff.GetBytes(), altbuffer, aProps.bufferSize());
if (!SendDeliverFrame(cap_engine, cap_id,
shMemBuff.Get(), size,
time_stamp, ntp_time, render_time)) {
if (!SendDeliverFrame(capEng, aStreamId,
shMemBuff.Get(), aProps)) {
return -1;
}
} else {
MOZ_ASSERT(buffer.Valid());
// ShmemBuffer was available, we're all good. A single copy happened
// in the original webrtc callback.
if (!SendDeliverFrame(cap_engine, cap_id,
buffer.Get(), size,
time_stamp, ntp_time, render_time)) {
if (!SendDeliverFrame(capEng, aStreamId,
buffer.Get(), aProps)) {
return -1;
}
}
@ -309,16 +252,16 @@ CamerasParent::GetBuffer(size_t aSize)
return mShmemPool.GetIfAvailable(aSize);
}
int
CallbackHelper::DeliverFrame(unsigned char* buffer,
size_t size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle)
int32_t
CallbackHelper::RenderFrame(uint32_t aStreamId, const webrtc::VideoFrame& aVideoFrame)
{
LOG_VERBOSE((__PRETTY_FUNCTION__));
RefPtr<DeliverFrameRunnable> runnable = nullptr;
// Get frame properties
camera::VideoFrameProperties properties;
VideoFrameUtils::InitFrameBufferProperties(aVideoFrame, properties);
// Get a shared memory buffer to copy the frame data into
ShmemBuffer shMemBuffer = mParent->GetBuffer(size);
ShmemBuffer shMemBuffer = mParent->GetBuffer(properties.bufferSize());
if (!shMemBuffer.Valid()) {
// Either we ran out of buffers or they're not the right size yet
LOG(("Correctly sized Video shmem not available in DeliverFrame"));
@ -326,30 +269,33 @@ CallbackHelper::DeliverFrame(unsigned char* buffer,
// the DeliverFrameRunnable constructor.
} else {
// Shared memory buffers of the right size are available, do the copy here.
memcpy(shMemBuffer.GetBytes(), buffer, size);
// Mark the original buffer as cleared.
buffer = nullptr;
VideoFrameUtils::CopyVideoFrameBuffers(shMemBuffer.GetBytes(),
properties.bufferSize(), aVideoFrame);
runnable = new DeliverFrameRunnable(mParent, mCapEngine, mStreamId,
Move(shMemBuffer), properties);
}
if (!runnable.get()) {
runnable = new DeliverFrameRunnable(mParent, mCapEngine, mStreamId,
aVideoFrame, properties);
}
RefPtr<DeliverFrameRunnable> runnable =
new DeliverFrameRunnable(mParent, mCapEngine, mCapturerId,
Move(shMemBuffer), buffer, size, time_stamp,
ntp_time, render_time);
MOZ_ASSERT(mParent);
nsIThread* thread = mParent->GetBackgroundThread();
MOZ_ASSERT(thread != nullptr);
thread->Dispatch(runnable, NS_DISPATCH_NORMAL);
return 0;
}
// XXX!!! FIX THIS -- we should move to pure DeliverI420Frame
int
CallbackHelper::DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame)
void
CallbackHelper::OnIncomingCapturedFrame(const int32_t id, const webrtc::VideoFrame& aVideoFrame)
{
return DeliverFrame(const_cast<uint8_t*>(webrtc_frame.buffer(webrtc::kYPlane)),
CalcBufferSize(webrtc::kI420, webrtc_frame.width(), webrtc_frame.height()),
webrtc_frame.timestamp(),
webrtc_frame.ntp_time_ms(),
webrtc_frame.render_time_ms(),
(void*) webrtc_frame.native_handle());
LOG_VERBOSE((__PRETTY_FUNCTION__));
RenderFrame(id,aVideoFrame);
}
void
CallbackHelper::OnCaptureDelayChanged(const int32_t id, const int32_t delay)
{
LOG((__PRETTY_FUNCTION__));
}
bool
@ -361,15 +307,17 @@ CamerasParent::RecvReleaseFrame(mozilla::ipc::Shmem&& s) {
bool
CamerasParent::SetupEngine(CaptureEngine aCapEngine)
{
LOG((__PRETTY_FUNCTION__));
MOZ_ASSERT(mVideoCaptureThread->thread_id() == PlatformThread::CurrentId());
EngineHelper *helper = &mEngines[aCapEngine];
RefPtr<mozilla::camera::VideoEngine>* engine = &mEngines[aCapEngine];
// Already initialized
if (helper->mEngine) {
if (engine->get()) {
return true;
}
webrtc::CaptureDeviceInfo *captureDeviceInfo = nullptr;
UniquePtr<webrtc::Config> config(new webrtc::Config);
switch (aCapEngine) {
case ScreenEngine:
@ -398,43 +346,19 @@ CamerasParent::SetupEngine(CaptureEngine aCapEngine)
break;
}
helper->mConfig.Set<webrtc::CaptureDeviceInfo>(captureDeviceInfo);
helper->mEngine = webrtc::VideoEngine::Create(helper->mConfig);
config->Set<webrtc::CaptureDeviceInfo>(captureDeviceInfo);
*engine = mozilla::camera::VideoEngine::Create(UniquePtr<const webrtc::Config>(config.release()));
if (!helper->mEngine) {
if (!engine->get()) {
LOG(("VideoEngine::Create failed"));
return false;
}
helper->mPtrViEBase = webrtc::ViEBase::GetInterface(helper->mEngine);
if (!helper->mPtrViEBase) {
LOG(("ViEBase::GetInterface failed"));
return false;
}
if (helper->mPtrViEBase->Init() < 0) {
LOG(("ViEBase::Init failed"));
return false;
}
helper->mPtrViECapture = webrtc::ViECapture::GetInterface(helper->mEngine);
if (!helper->mPtrViECapture) {
LOG(("ViECapture::GetInterface failed"));
return false;
}
RefPtr<InputObserver>* observer = mObservers.AppendElement(new InputObserver(this));
#ifdef DEBUG
MOZ_ASSERT(0 == helper->mPtrViECapture->RegisterInputObserver(observer->get()));
#else
helper->mPtrViECapture->RegisterInputObserver(observer->get());
#endif
helper->mPtrViERender = webrtc::ViERender::GetInterface(helper->mEngine);
if (!helper->mPtrViERender) {
LOG(("ViERender::GetInterface failed"));
return false;
auto device_info = engine->get()->GetOrCreateVideoCaptureDeviceInfo();
MOZ_ASSERT(device_info);
if (device_info) {
device_info->RegisterVideoInputFeedBack(**observer);
}
return true;
@ -452,38 +376,25 @@ CamerasParent::CloseEngines()
// Stop the callers
while (mCallbacks.Length()) {
auto capEngine = mCallbacks[0]->mCapEngine;
auto capNum = mCallbacks[0]->mCapturerId;
LOG(("Forcing shutdown of engine %d, capturer %d", capEngine, capNum));
StopCapture(capEngine, capNum);
Unused << ReleaseCaptureDevice(capEngine, capNum);
auto streamNum = mCallbacks[0]->mStreamId;
LOG(("Forcing shutdown of engine %d, capturer %d", capEngine, streamNum));
StopCapture(capEngine, streamNum);
Unused << ReleaseCaptureDevice(capEngine, streamNum);
}
for (int i = 0; i < CaptureEngine::MaxEngine; i++) {
if (mEngines[i].mEngineIsRunning) {
LOG(("Being closed down while engine %d is running!", i));
}
if (mEngines[i].mPtrViERender) {
mEngines[i].mPtrViERender->Release();
mEngines[i].mPtrViERender = nullptr;
}
if (mEngines[i].mPtrViECapture) {
#ifdef DEBUG
MOZ_ASSERT(0 == mEngines[i].mPtrViECapture->DeregisterInputObserver());
#else
mEngines[i].mPtrViECapture->DeregisterInputObserver();
#endif
if (auto engine = mEngines[i].get() ){
if (engine->IsRunning()) {
LOG(("Being closed down while engine %d is running!", i));
}
mEngines[i].mPtrViECapture->Release();
mEngines[i].mPtrViECapture = nullptr;
}
if(mEngines[i].mPtrViEBase) {
mEngines[i].mPtrViEBase->Release();
mEngines[i].mPtrViEBase = nullptr;
}
if (mEngines[i].mEngine) {
mEngines[i].mEngine->SetTraceCallback(nullptr);
webrtc::VideoEngine::Delete(mEngines[i].mEngine);
mEngines[i].mEngine = nullptr;
auto device_info = engine->GetOrCreateVideoCaptureDeviceInfo();
MOZ_ASSERT(device_info);
if (device_info) {
device_info->DeRegisterVideoInputFeedBack();
}
mozilla::camera::VideoEngine::Delete(engine);
mEngines[i] = nullptr;
}
}
@ -492,21 +403,21 @@ CamerasParent::CloseEngines()
mWebRTCAlive = false;
}
bool
VideoEngine *
CamerasParent::EnsureInitialized(int aEngine)
{
LOG((__PRETTY_FUNCTION__));
LOG_VERBOSE((__PRETTY_FUNCTION__));
// We're shutting down, don't try to do new WebRTC ops.
if (!mWebRTCAlive) {
return false;
return nullptr;
}
CaptureEngine capEngine = static_cast<CaptureEngine>(aEngine);
if (!SetupEngine(capEngine)) {
LOG(("CamerasParent failed to initialize engine"));
return false;
return nullptr;
}
return true;
return mEngines[aEngine];
}
// Dispatch the runnable to do the camera operation on the
@ -518,13 +429,15 @@ bool
CamerasParent::RecvNumberOfCaptureDevices(const CaptureEngine& aCapEngine)
{
LOG((__PRETTY_FUNCTION__));
LOG(("CaptureEngine=%d", aCapEngine));
RefPtr<CamerasParent> self(this);
RefPtr<Runnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine]() -> nsresult {
int num = -1;
if (self->EnsureInitialized(aCapEngine)) {
num = self->mEngines[aCapEngine].mPtrViECapture->NumberOfCaptureDevices();
if (auto engine = self->EnsureInitialized(aCapEngine)) {
if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) {
num = devInfo->NumberOfDevices();
}
}
RefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, num]() -> nsresult {
@ -591,11 +504,10 @@ CamerasParent::RecvNumberOfCapabilities(const CaptureEngine& aCapEngine,
RefPtr<Runnable> webrtc_runnable =
media::NewRunnableFrom([self, unique_id, aCapEngine]() -> nsresult {
int num = -1;
if (self->EnsureInitialized(aCapEngine)) {
num =
self->mEngines[aCapEngine].mPtrViECapture->NumberOfCapabilities(
unique_id.get(),
MediaEngineSource::kMaxUniqueIdLength);
if (auto engine = self->EnsureInitialized(aCapEngine)) {
if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) {
num = devInfo->NumberOfCapabilities(unique_id.get());
}
}
RefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, num]() -> nsresult {
@ -630,18 +542,19 @@ CamerasParent::RecvGetCaptureCapability(const CaptureEngine& aCapEngine,
RefPtr<CamerasParent> self(this);
RefPtr<Runnable> webrtc_runnable =
media::NewRunnableFrom([self, unique_id, aCapEngine, num]() -> nsresult {
webrtc::CaptureCapability webrtcCaps;
webrtc::VideoCaptureCapability webrtcCaps;
int error = -1;
if (self->EnsureInitialized(aCapEngine)) {
error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureCapability(
unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, num, webrtcCaps);
if (auto engine = self->EnsureInitialized(aCapEngine)) {
if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()){
error = devInfo->GetCapability(unique_id.get(), num, webrtcCaps);
}
}
RefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, webrtcCaps, error]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
CaptureCapability capCap(webrtcCaps.width,
VideoCaptureCapability capCap(webrtcCaps.width,
webrtcCaps.height,
webrtcCaps.maxFPS,
webrtcCaps.expectedCaptureDelay,
@ -682,15 +595,15 @@ CamerasParent::RecvGetCaptureDevice(const CaptureEngine& aCapEngine,
char deviceUniqueId[MediaEngineSource::kMaxUniqueIdLength];
nsCString name;
nsCString uniqueId;
int devicePid = 0;
pid_t devicePid = 0;
int error = -1;
if (self->EnsureInitialized(aCapEngine)) {
error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureDevice(aListNumber,
deviceName,
sizeof(deviceName),
deviceUniqueId,
sizeof(deviceUniqueId),
&devicePid);
if (auto engine = self->EnsureInitialized(aCapEngine)) {
if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) {
error = devInfo->GetDeviceName(aListNumber, deviceName, sizeof(deviceName),
deviceUniqueId, sizeof(deviceUniqueId),
nullptr, 0,
&devicePid);
}
}
if (!error) {
name.Assign(deviceName);
@ -806,8 +719,17 @@ CamerasParent::RecvAllocateCaptureDevice(const CaptureEngine& aCapEngine,
int numdev = -1;
int error = -1;
if (allowed && self->EnsureInitialized(aCapEngine)) {
error = self->mEngines[aCapEngine].mPtrViECapture->AllocateCaptureDevice(
unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, numdev);
auto engine = self->mEngines[aCapEngine].get();
engine->CreateVideoCapture(numdev, unique_id.get());
engine->WithEntry(numdev, [&error](VideoEngine::CaptureEntry& cap) {
if (cap.VideoCapture()) {
if (!cap.VideoRenderer()) {
LOG(("VideoEngine::VideoRenderer() failed"));
} else {
error = 0;
}
}
});
}
RefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, numdev, error]() -> nsresult {
@ -838,8 +760,8 @@ CamerasParent::ReleaseCaptureDevice(const CaptureEngine& aCapEngine,
const int& capnum)
{
int error = -1;
if (EnsureInitialized(aCapEngine)) {
error = mEngines[aCapEngine].mPtrViECapture->ReleaseCaptureDevice(capnum);
if (auto engine = EnsureInitialized(aCapEngine)) {
error = engine->ReleaseVideoCapture(capnum);
}
return error;
}
@ -881,44 +803,48 @@ CamerasParent::RecvReleaseCaptureDevice(const CaptureEngine& aCapEngine,
bool
CamerasParent::RecvStartCapture(const CaptureEngine& aCapEngine,
const int& capnum,
const CaptureCapability& ipcCaps)
const VideoCaptureCapability& ipcCaps)
{
LOG((__PRETTY_FUNCTION__));
RefPtr<CamerasParent> self(this);
RefPtr<Runnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine, capnum, ipcCaps]() -> nsresult {
LOG((__PRETTY_FUNCTION__));
CallbackHelper** cbh;
webrtc::ExternalRenderer* render;
EngineHelper* helper = nullptr;
webrtc::VideoRenderCallback* render;
VideoEngine* engine = nullptr;
int error = -1;
if (self->EnsureInitialized(aCapEngine)) {
cbh = self->mCallbacks.AppendElement(
new CallbackHelper(static_cast<CaptureEngine>(aCapEngine), capnum, self));
render = static_cast<webrtc::ExternalRenderer*>(*cbh);
render = static_cast<webrtc::VideoRenderCallback*>(*cbh);
helper = &self->mEngines[aCapEngine];
error =
helper->mPtrViERender->AddRenderer(capnum, webrtc::kVideoI420, render);
if (!error) {
error = helper->mPtrViERender->StartRender(capnum);
}
engine = self->mEngines[aCapEngine];
engine->WithEntry(capnum, [capnum, &render, &engine, &error, &ipcCaps, &cbh](VideoEngine::CaptureEntry& cap) {
cap.VideoRenderer()->AddIncomingRenderStream(capnum,0, 0., 0., 1., 1.);
error = cap.VideoRenderer()->AddExternalRenderCallback(capnum, render);
if (!error) {
error = cap.VideoRenderer()->StartRender(capnum);
}
webrtc::CaptureCapability capability;
capability.width = ipcCaps.width();
capability.height = ipcCaps.height();
capability.maxFPS = ipcCaps.maxFPS();
capability.expectedCaptureDelay = ipcCaps.expectedCaptureDelay();
capability.rawType = static_cast<webrtc::RawVideoType>(ipcCaps.rawType());
capability.codecType = static_cast<webrtc::VideoCodecType>(ipcCaps.codecType());
capability.interlaced = ipcCaps.interlaced();
webrtc::VideoCaptureCapability capability;
capability.width = ipcCaps.width();
capability.height = ipcCaps.height();
capability.maxFPS = ipcCaps.maxFPS();
capability.expectedCaptureDelay = ipcCaps.expectedCaptureDelay();
capability.rawType = static_cast<webrtc::RawVideoType>(ipcCaps.rawType());
capability.codecType = static_cast<webrtc::VideoCodecType>(ipcCaps.codecType());
capability.interlaced = ipcCaps.interlaced();
if (!error) {
error = helper->mPtrViECapture->StartCapture(capnum, capability);
}
if (!error) {
helper->mEngineIsRunning = true;
}
if (!error) {
error = cap.VideoCapture()->StartCapture(capability);
}
if (!error) {
engine->Startup();
cap.VideoCapture()->RegisterCaptureDataCallback(*static_cast<webrtc::VideoCaptureDataCallback*>(*cbh));
}
});
}
RefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, error]() -> nsresult {
@ -944,20 +870,27 @@ void
CamerasParent::StopCapture(const CaptureEngine& aCapEngine,
const int& capnum)
{
if (EnsureInitialized(aCapEngine)) {
mEngines[aCapEngine].mPtrViECapture->StopCapture(capnum);
mEngines[aCapEngine].mPtrViERender->StopRender(capnum);
mEngines[aCapEngine].mPtrViERender->RemoveRenderer(capnum);
mEngines[aCapEngine].mEngineIsRunning = false;
for (size_t i = 0; i < mCallbacks.Length(); i++) {
if (mCallbacks[i]->mCapEngine == aCapEngine
&& mCallbacks[i]->mCapturerId == capnum) {
delete mCallbacks[i];
mCallbacks.RemoveElementAt(i);
if (auto engine = EnsureInitialized(aCapEngine)) {
engine->WithEntry(capnum,[capnum](VideoEngine::CaptureEntry& cap){
if (cap.VideoCapture()) {
cap.VideoCapture()->StopCapture();
cap.VideoCapture()->DeRegisterCaptureDataCallback();
}
if (cap.VideoRenderer()) {
cap.VideoRenderer()->StopRender(capnum);
}
});
// we're removing elements, iterate backwards
for (size_t i = mCallbacks.Length(); i > 0; i--) {
if (mCallbacks[i-1]->mCapEngine == aCapEngine
&& mCallbacks[i-1]->mStreamId == (uint32_t) capnum) {
delete mCallbacks[i-1];
mCallbacks.RemoveElementAt(i-1);
break;
}
}
engine->RemoveRenderer(capnum);
engine->Shutdown();
}
}
@ -1077,7 +1010,7 @@ CamerasParent::~CamerasParent()
// That runnable takes a ref to us, so it must have finished
// by the time we get here.
for (int i = 0; i < CaptureEngine::MaxEngine; i++) {
MOZ_ASSERT(!mEngines[i].mEngine);
MOZ_ASSERT(!mEngines[i]);
}
#endif
}

76
dom/media/systemservices/CamerasParent.h
View file

@ -7,19 +7,21 @@
#define mozilla_CamerasParent_h
#include "nsIObserver.h"
#include "VideoEngine.h"
#include "mozilla/dom/ContentParent.h"
#include "mozilla/camera/PCamerasParent.h"
#include "mozilla/ipc/Shmem.h"
#include "mozilla/ShmemPool.h"
#include "mozilla/Atomics.h"
#include "webrtc/modules/video_capture/video_capture.h"
#include "webrtc/modules/video_render/video_render_impl.h"
#include "webrtc/modules/video_capture/video_capture_defines.h"
#include "webrtc/common_video/include/incoming_video_stream.h"
// conflicts with #include of scoped_ptr.h
#undef FF
#include "webrtc/common.h"
// Video Engine
#include "webrtc/video_engine/include/vie_base.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "webrtc/video_engine/include/vie_render.h"
#include "CamerasChild.h"
#include "base/thread.h"
@ -29,60 +31,43 @@ namespace camera {
class CamerasParent;
class CallbackHelper : public webrtc::ExternalRenderer
class CallbackHelper :
public webrtc::VideoRenderCallback,
public webrtc::VideoCaptureDataCallback
{
public:
CallbackHelper(CaptureEngine aCapEng, int aCapId, CamerasParent *aParent)
: mCapEngine(aCapEng), mCapturerId(aCapId), mParent(aParent) {};
CallbackHelper(CaptureEngine aCapEng, uint32_t aStreamId, CamerasParent *aParent)
: mCapEngine(aCapEng), mStreamId(aStreamId), mParent(aParent) {};
// ViEExternalRenderer implementation. These callbacks end up
// running on the VideoCapture thread.
virtual int FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams) override;
virtual int DeliverFrame(unsigned char* buffer,
size_t size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle) override;
virtual int DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame) override;
virtual bool IsTextureSupported() override { return false; };
virtual int32_t RenderFrame(const uint32_t aStreamId, const webrtc::VideoFrame& video_frame) override;
// From VideoCaptureCallback
virtual void OnIncomingCapturedFrame(const int32_t id, const webrtc::VideoFrame& videoFrame) override;
virtual void OnCaptureDelayChanged(const int32_t id, const int32_t delay) override;
// TODO(@@NG) This is now part of webrtc::VideoRenderer, not in the webrtc::VideoRenderCallback
// virtual bool IsTextureSupported() const override { return false; };
//
// virtual bool SmoothsRenderedFrames() const override { return false; }
friend CamerasParent;
private:
CaptureEngine mCapEngine;
int mCapturerId;
uint32_t mStreamId;
CamerasParent *mParent;
};
class EngineHelper
{
public:
EngineHelper() :
mEngine(nullptr), mPtrViEBase(nullptr), mPtrViECapture(nullptr),
mPtrViERender(nullptr), mEngineIsRunning(false) {};
webrtc::VideoEngine *mEngine;
webrtc::ViEBase *mPtrViEBase;
webrtc::ViECapture *mPtrViECapture;
webrtc::ViERender *mPtrViERender;
// The webrtc code keeps a reference to this one.
webrtc::Config mConfig;
// Engine alive
bool mEngineIsRunning;
};
class InputObserver : public webrtc::ViEInputObserver
class InputObserver : public webrtc::VideoInputFeedBack
{
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(InputObserver)
explicit InputObserver(CamerasParent* aParent)
: mParent(aParent) {};
virtual void DeviceChange();
virtual void OnDeviceChange();
friend CamerasParent;
@ -113,7 +98,7 @@ public:
const int&) override;
virtual bool RecvGetCaptureDevice(const CaptureEngine&, const int&) override;
virtual bool RecvStartCapture(const CaptureEngine&, const int&,
const CaptureCapability&) override;
const VideoCaptureCapability&) override;
virtual bool RecvStopCapture(const CaptureEngine&, const int&) override;
virtual bool RecvReleaseFrame(mozilla::ipc::Shmem&&) override;
virtual bool RecvAllDone() override;
@ -128,13 +113,10 @@ public:
// helper to forward to the PBackground thread
int DeliverFrameOverIPC(CaptureEngine capEng,
int cap_id,
uint32_t aStreamId,
ShmemBuffer buffer,
unsigned char* altbuffer,
size_t size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time);
VideoFrameProperties& aProps);
CamerasParent();
@ -147,14 +129,14 @@ protected:
int ReleaseCaptureDevice(const CaptureEngine& aCapEngine, const int& capnum);
bool SetupEngine(CaptureEngine aCapEngine);
bool EnsureInitialized(int aEngine);
VideoEngine* EnsureInitialized(int aEngine);
void CloseEngines();
void StopIPC();
void StopVideoCapture();
// Can't take already_AddRefed because it can fail in stupid ways.
nsresult DispatchToVideoCaptureThread(Runnable* event);
EngineHelper mEngines[CaptureEngine::MaxEngine];
RefPtr<VideoEngine> mEngines[CaptureEngine::MaxEngine];
nsTArray<CallbackHelper*> mCallbacks;
// image buffers

19
dom/media/systemservices/LoadManager.h
View file

@ -14,8 +14,8 @@
#include "nsIObserver.h"
#include "webrtc/common_types.h"
#include "webrtc/video_engine/include/vie_base.h"
#include "webrtc/call.h"
#include "webrtc/video/overuse_frame_detector.h"
extern mozilla::LazyLogModule gLoadManagerLog;
namespace mozilla {
@ -77,7 +77,7 @@ private:
};
class LoadManager final : public webrtc::CPULoadStateCallbackInvoker,
public webrtc::CpuOveruseObserver
public webrtc::LoadObserver
{
public:
explicit LoadManager(LoadManagerSingleton* aManager)
@ -93,13 +93,14 @@ public:
{
mManager->RemoveObserver(aObserver);
}
void OveruseDetected() override
void OnLoadUpdate(webrtc::LoadObserver::Load load_state) override
{
mManager->OveruseDetected();
}
void NormalUsage() override
{
mManager->NormalUsage();
if (load_state == webrtc::LoadObserver::kOveruse) {
mManager->OveruseDetected();
} else if (load_state == webrtc::LoadObserver::kUnderuse) {
mManager->NormalUsage();
}
}
private:

41
dom/media/systemservices/PCameras.ipdl
View file

@ -10,7 +10,8 @@ using mozilla::camera::CaptureEngine from "mozilla/media/CamerasTypes.h";
namespace mozilla {
namespace camera {
struct CaptureCapability
// IPC analog for webrtc::VideoCaptureCapability
struct VideoCaptureCapability
{
int width;
int height;
@ -21,6 +22,32 @@ struct CaptureCapability
bool interlaced;
};
// IPC analog for webrtc::VideoFrame
// the described buffer is transported seperately in a Shmem
// See VideoFrameUtils.h
struct VideoFrameProperties
{
// Size of image data within the ShMem,
// the ShMem is at least this large
uint32_t bufferSize;
// From webrtc::VideoFrame
uint32_t timeStamp;
int64_t ntpTimeMs;
int64_t renderTimeMs;
// See webrtc/**/rotation.h
int rotation;
int yAllocatedSize;
int uAllocatedSize;
int vAllocatedSize;
// From webrtc::VideoFrameBuffer
int width;
int height;
int yStride;
int uStride;
int vStride;
};
async protocol PCameras
{
manager PBackground;
@ -28,14 +55,13 @@ async protocol PCameras
child:
async FrameSizeChange(CaptureEngine capEngine, int cap_id, int w, int h);
// transfers ownership of |buffer| from parent to child
async DeliverFrame(CaptureEngine capEngine, int cap_id,
Shmem buffer, size_t size, uint32_t time_stamp,
int64_t ntp_time, int64_t render_time);
async DeliverFrame(CaptureEngine capEngine, int streamId,
Shmem buffer, VideoFrameProperties props);
async DeviceChange();
async ReplyNumberOfCaptureDevices(int numdev);
async ReplyNumberOfCapabilities(int numdev);
async ReplyAllocateCaptureDevice(int numdev);
async ReplyGetCaptureCapability(CaptureCapability cap);
async ReplyGetCaptureCapability(VideoCaptureCapability cap);
async ReplyGetCaptureDevice(nsCString device_name, nsCString device_id, bool scary);
async ReplyFailure();
async ReplySuccess();
@ -45,12 +71,13 @@ parent:
async NumberOfCaptureDevices(CaptureEngine engine);
async NumberOfCapabilities(CaptureEngine engine, nsCString deviceUniqueIdUTF8);
async GetCaptureCapability(CaptureEngine engine, nsCString unique_idUTF8, int capability_number);
async GetCaptureCapability(CaptureEngine engine, nsCString unique_idUTF8,
int capability_number);
async GetCaptureDevice(CaptureEngine engine, int num);
async AllocateCaptureDevice(CaptureEngine engine, nsCString unique_idUTF8, nsCString origin);
async ReleaseCaptureDevice(CaptureEngine engine, int numdev);
async StartCapture(CaptureEngine engine, int numdev, CaptureCapability capability);
async StartCapture(CaptureEngine engine, int numdev, VideoCaptureCapability capability);
async StopCapture(CaptureEngine engine, int numdev);
// transfers frame back
async ReleaseFrame(Shmem s);

4
dom/media/systemservices/ShmemPool.cpp
View file

@ -39,7 +39,7 @@ mozilla::ShmemBuffer ShmemPool::GetIfAvailable(size_t aSize)
MOZ_ASSERT(res.mShmem.IsWritable(), "Pool in Shmem is not writable?");
if (res.mShmem.Size<char>() < aSize) {
if (res.mShmem.Size<uint8_t>() < aSize) {
LOG(("Free Shmem but not of the right size"));
return ShmemBuffer();
}
@ -64,7 +64,7 @@ void ShmemPool::Put(ShmemBuffer&& aShmem)
#ifdef DEBUG
size_t poolUse = mShmemPool.Length() - mPoolFree;
if (poolUse > 0) {
LOG(("ShmemPool usage reduced to %d buffers", poolUse));
LOG_VERBOSE(("ShmemPool usage reduced to %d buffers", poolUse));
}
#endif
}

4
dom/media/systemservices/ShmemPool.h
View file

@ -47,8 +47,8 @@ public:
return mInitialized;
}
char* GetBytes() {
return mShmem.get<char>();
uint8_t * GetBytes() {
return mShmem.get<uint8_t>();
}
mozilla::ipc::Shmem& Get() {

185
dom/media/systemservices/VideoEngine.cpp Normal file
View file

@ -0,0 +1,185 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "VideoEngine.h"
#include "webrtc/video_engine/browser_capture_impl.h"
#ifdef WEBRTC_ANDROID
#include "webrtc/modules/video_capture/video_capture.h"
#ifdef WEBRTC_INCLUDE_INTERNAL_VIDEO_RENDER
#include "webrtc/modules/video_render/video_render.h"
#endif
#endif
namespace mozilla {
namespace camera {
#undef LOG
#undef LOG_ENABLED
mozilla::LazyLogModule gVideoEngineLog("VideoEngine");
#define LOG(args) MOZ_LOG(gVideoEngineLog, mozilla::LogLevel::Debug, args)
#define LOG_ENABLED() MOZ_LOG_TEST(gVideoEngineLog, mozilla::LogLevel::Debug)
int VideoEngine::sId = 0;
#if defined(ANDROID)
int VideoEngine::SetAndroidObjects(JavaVM* javaVM) {
LOG((__PRETTY_FUNCTION__));
if (webrtc::SetCaptureAndroidVM(javaVM) != 0) {
LOG(("Could not set capture Android VM"));
return -1;
}
#ifdef WEBRTC_INCLUDE_INTERNAL_VIDEO_RENDER
if (webrtc::SetRenderAndroidVM(javaVM) != 0) {
LOG(("Could not set render Android VM"));
return -1;
}
#endif
return 0;
}
#endif
void
VideoEngine::CreateVideoCapture(int32_t& id, const char* deviceUniqueIdUTF8) {
LOG((__PRETTY_FUNCTION__));
id = GenerateId();
LOG(("CaptureDeviceInfo.type=%s id=%d",mCaptureDevInfo.TypeName(),id));
CaptureEntry entry = {-1,nullptr,nullptr};
if (mCaptureDevInfo.type == webrtc::CaptureDeviceType::Camera) {
entry = CaptureEntry(id,
webrtc::VideoCaptureFactory::Create(id, deviceUniqueIdUTF8),
nullptr);
} else {
#ifndef WEBRTC_ANDROID
entry = CaptureEntry(
id,
webrtc::DesktopCaptureImpl::Create(id, deviceUniqueIdUTF8, mCaptureDevInfo.type),
nullptr);
#else
MOZ_ASSERT("CreateVideoCapture NO DESKTOP CAPTURE IMPL ON ANDROID" == nullptr);
#endif
}
mCaps.emplace(id,std::move(entry));
}
int
VideoEngine::ReleaseVideoCapture(const int32_t id) {
bool found = false;
WithEntry(id, [&found](CaptureEntry& cap) {
cap.mVideoCaptureModule = nullptr;
found = true;
});
return found ? 0 : (-1);
}
std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo>
VideoEngine::GetOrCreateVideoCaptureDeviceInfo() {
if (mDeviceInfo) {
return mDeviceInfo;
}
switch (mCaptureDevInfo.type) {
case webrtc::CaptureDeviceType::Camera: {
mDeviceInfo.reset(webrtc::VideoCaptureFactory::CreateDeviceInfo(0));
break;
}
case webrtc::CaptureDeviceType::Browser: {
mDeviceInfo.reset(webrtc::BrowserDeviceInfoImpl::CreateDeviceInfo());
break;
}
// Window, Application, and Screen types are handled by DesktopCapture
case webrtc::CaptureDeviceType::Window:
case webrtc::CaptureDeviceType::Application:
case webrtc::CaptureDeviceType::Screen: {
#if !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS)
mDeviceInfo.reset(webrtc::DesktopCaptureImpl::CreateDeviceInfo(mId,mCaptureDevInfo.type));
#else
MOZ_ASSERT("GetVideoCaptureDeviceInfo NO DESKTOP CAPTURE IMPL ON ANDROID" == nullptr);
mDeviceInfo.reset();
#endif
break;
}
}
return mDeviceInfo;
}
void
VideoEngine::RemoveRenderer(int capnum) {
WithEntry(capnum, [](CaptureEntry& cap) {
cap.mVideoRender = nullptr;
});
}
const UniquePtr<const webrtc::Config>&
VideoEngine::GetConfiguration() {
return mConfig;
}
RefPtr<VideoEngine> VideoEngine::Create(UniquePtr<const webrtc::Config>&& aConfig) {
LOG((__PRETTY_FUNCTION__));
LOG(("Creating new VideoEngine with CaptureDeviceType %s",
aConfig->Get<webrtc::CaptureDeviceInfo>().TypeName()));
RefPtr<VideoEngine> engine(new VideoEngine(std::move(aConfig)));
return engine;
}
VideoEngine::CaptureEntry::CaptureEntry(int32_t aCapnum,
rtc::scoped_refptr<webrtc::VideoCaptureModule> aCapture,
webrtc::VideoRender * aRenderer):
mCapnum(aCapnum),
mVideoCaptureModule(aCapture),
mVideoRender(aRenderer)
{}
rtc::scoped_refptr<webrtc::VideoCaptureModule>
VideoEngine::CaptureEntry::VideoCapture() {
return mVideoCaptureModule;
}
const UniquePtr<webrtc::VideoRender>&
VideoEngine::CaptureEntry::VideoRenderer() {
if (!mVideoRender) {
MOZ_ASSERT(mCapnum != -1);
// Create a VideoRender on demand
mVideoRender = UniquePtr<webrtc::VideoRender>(
webrtc::VideoRender::CreateVideoRender(mCapnum,nullptr,false,webrtc::kRenderExternal));
}
return mVideoRender;
}
int32_t
VideoEngine::CaptureEntry::Capnum() const {
return mCapnum;
}
bool VideoEngine::WithEntry(const int32_t entryCapnum,
const std::function<void(CaptureEntry &entry)>&& fn) {
auto it = mCaps.find(entryCapnum);
if (it == mCaps.end()) {
return false;
}
fn(it->second);
return true;
}
int32_t
VideoEngine::GenerateId() {
// XXX Something better than this (a map perhaps, or a simple boolean TArray, given
// the number in-use is O(1) normally!)
return mId = sId++;
}
VideoEngine::VideoEngine(UniquePtr<const webrtc::Config>&& aConfig):
mCaptureDevInfo(aConfig->Get<webrtc::CaptureDeviceInfo>()),
mDeviceInfo(nullptr),
mConfig(std::move(aConfig))
{
LOG((__PRETTY_FUNCTION__));
}
}
}

104
dom/media/systemservices/VideoEngine.h Normal file
View file

@ -0,0 +1,104 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_VideoEngine_h
#define mozilla_VideoEngine_h
#include "MediaEngine.h"
#include "VideoFrameUtils.h"
#include "mozilla/media/MediaUtils.h"
#include "webrtc/common.h"
#include "webrtc/modules/video_capture/video_capture_impl.h"
#include "webrtc/modules/video_render/video_render.h"
#include "webrtc/modules/video_capture/video_capture_defines.h"
#include "webrtc/modules/video_capture/video_capture_factory.h"
#include "webrtc/video_engine/desktop_capture_impl.h"
#include <memory>
#include <functional>
namespace mozilla {
namespace camera {
// Historically the video engine was part of webrtc
// it was removed (and reimplemented in Talk)
class VideoEngine
{
private:
virtual ~VideoEngine (){};
public:
VideoEngine (){};
NS_INLINE_DECL_REFCOUNTING(VideoEngine)
static RefPtr<VideoEngine> Create(UniquePtr<const webrtc::Config>&& aConfig);
#if defined(ANDROID)
static int SetAndroidObjects(JavaVM* javaVM);
#endif
void CreateVideoCapture(int32_t& id, const char* deviceUniqueIdUTF8);
int ReleaseVideoCapture(const int32_t id);
// VideoEngine is responsible for any cleanup in its modules
static void Delete(VideoEngine * engine) { }
/** Returns or creates a new new DeviceInfo.
* It is cached to prevent repeated lengthy polling for "realness"
* of the hardware devices. This could be handled in a more elegant
* way in the future.
* @return on failure the shared_ptr will be null, otherwise it will contain a DeviceInfo.
* @see bug 1305212 https://bugzilla.mozilla.org/show_bug.cgi?id=1305212
*/
std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> GetOrCreateVideoCaptureDeviceInfo();
void RemoveRenderer(int capnum);
const UniquePtr<const webrtc::Config>& GetConfiguration();
void Startup() {
mIsRunning = true;
}
void Shutdown() {
mIsRunning = false;
}
bool IsRunning() const {
return mIsRunning;
}
class CaptureEntry {
public:
CaptureEntry(int32_t aCapnum,
rtc::scoped_refptr<webrtc::VideoCaptureModule> aCapture,
webrtc::VideoRender* aRenderer);
int32_t Capnum() const;
rtc::scoped_refptr<webrtc::VideoCaptureModule> VideoCapture();
const UniquePtr<webrtc::VideoRender> & VideoRenderer();
private:
int32_t mCapnum;
rtc::scoped_refptr<webrtc::VideoCaptureModule> mVideoCaptureModule;
UniquePtr<webrtc::VideoRender> mVideoRender;
friend class VideoEngine;
};
// Returns true iff an entry for capnum exists
bool WithEntry(const int32_t entryCapnum, const std::function<void(CaptureEntry &entry)>&& fn);
private:
explicit VideoEngine(UniquePtr<const webrtc::Config>&& aConfig);
bool mIsRunning;
int32_t mId;
webrtc::CaptureDeviceInfo mCaptureDevInfo;
std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> mDeviceInfo;
UniquePtr<const webrtc::Config> mConfig;
std::map<int32_t, CaptureEntry> mCaps;
int32_t GenerateId();
static int32_t sId;
};
}
}
#endif

92
dom/media/systemservices/VideoFrameUtils.cpp Normal file
View file

@ -0,0 +1,92 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "VideoFrameUtils.h"
#include "webrtc/video_frame.h"
#include "mozilla/ShmemPool.h"
namespace mozilla {
size_t
VideoFrameUtils::TotalRequiredBufferSize(
const webrtc::VideoFrame& aVideoFrame)
{
static const webrtc::PlaneType kPlanes[] =
{webrtc::kYPlane, webrtc::kUPlane, webrtc::kVPlane};
if (aVideoFrame.IsZeroSize()) {
return 0;
}
size_t sum = 0;
for (auto plane : kPlanes) {
sum += aVideoFrame.allocated_size(plane);
}
return sum;
}
void VideoFrameUtils::InitFrameBufferProperties(
const webrtc::VideoFrame& aVideoFrame,
camera::VideoFrameProperties& aDestProps)
{
// The VideoFrameBuffer image data stored in the accompanying buffer
// the buffer is at least this size of larger.
aDestProps.bufferSize() = TotalRequiredBufferSize(aVideoFrame);
aDestProps.timeStamp() = aVideoFrame.timestamp();
aDestProps.ntpTimeMs() = aVideoFrame.ntp_time_ms();
aDestProps.renderTimeMs() = aVideoFrame.render_time_ms();
aDestProps.rotation() = aVideoFrame.rotation();
aDestProps.yAllocatedSize() = aVideoFrame.allocated_size(webrtc::kYPlane);
aDestProps.uAllocatedSize() = aVideoFrame.allocated_size(webrtc::kYPlane);
aDestProps.vAllocatedSize() = aVideoFrame.allocated_size(webrtc::kYPlane);
aDestProps.width() = aVideoFrame.width();
aDestProps.height() = aVideoFrame.height();
aDestProps.yStride() = aVideoFrame.stride(webrtc::kYPlane);
aDestProps.uStride() = aVideoFrame.stride(webrtc::kUPlane);
aDestProps.vStride() = aVideoFrame.stride(webrtc::kVPlane);
}
void VideoFrameUtils::CopyVideoFrameBuffers(uint8_t* aDestBuffer,
const size_t aDestBufferSize,
const webrtc::VideoFrame& aFrame)
{
static const webrtc::PlaneType planes[] = {webrtc::kYPlane, webrtc::kUPlane, webrtc::kVPlane};
size_t aggregateSize = TotalRequiredBufferSize(aFrame);
MOZ_ASSERT(aDestBufferSize >= aggregateSize);
// If planes are ordered YUV and contiguous then do a single copy
if ((aFrame.buffer(webrtc::kYPlane) != nullptr)
// Check that the three planes are ordered
&& (aFrame.buffer(webrtc::kYPlane) < aFrame.buffer(webrtc::kUPlane))
&& (aFrame.buffer(webrtc::kUPlane) < aFrame.buffer(webrtc::kVPlane))
// Check that the last plane ends at firstPlane[totalsize]
&& (&aFrame.buffer(webrtc::kYPlane)[aggregateSize] == &aFrame.buffer(webrtc::kVPlane)[aFrame.allocated_size(webrtc::kVPlane)]))
{
memcpy(aDestBuffer,aFrame.buffer(webrtc::kYPlane),aggregateSize);
return;
}
// Copy each plane
size_t offset = 0;
for (auto plane: planes) {
memcpy(&aDestBuffer[offset], aFrame.buffer(plane), aFrame.allocated_size(plane));
offset += aFrame.allocated_size(plane);
}
}
void VideoFrameUtils::CopyVideoFrameBuffers(ShmemBuffer& aDestShmem,
const webrtc::VideoFrame& aVideoFrame)
{
CopyVideoFrameBuffers(aDestShmem.Get().get<uint8_t>(), aDestShmem.Get().Size<uint8_t>(), aVideoFrame);
}
}

51
dom/media/systemservices/VideoFrameUtils.h Normal file
View file

@ -0,0 +1,51 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_VideoFrameUtil_h
#define mozilla_VideoFrameUtil_h
#include "mozilla/camera/PCameras.h"
namespace webrtc {
class VideoFrame;
}
namespace mozilla
{
class ShmemBuffer;
// Util methods for working with webrtc::VideoFrame(s) and
// the IPC classes that are used to deliver their contents to the
// MediaEnginge
class VideoFrameUtils {
public:
// Returns the total number of bytes necessary to copy a VideoFrame's buffer
// across all planes.
static size_t TotalRequiredBufferSize(const webrtc::VideoFrame & frame);
// Initializes a camera::VideoFrameProperties from a VideoFrameBuffer
static void InitFrameBufferProperties(const webrtc::VideoFrame& aVideoFrame,
camera::VideoFrameProperties & aDestProperties);
// Copies the buffers out of a VideoFrameBuffer into a buffer.
// Attempts to make as few memcopies as possible.
static void CopyVideoFrameBuffers(uint8_t * aDestBuffer,
const size_t aDestBufferSize,
const webrtc::VideoFrame & aVideoFrame);
// Copies the buffers in a VideoFrameBuffer into a Shmem
// returns the eno from the underlying memcpy.
static void CopyVideoFrameBuffers(ShmemBuffer & aDestShmem,
const webrtc::VideoFrame & aVideoFrame);
};
} /* namespace mozilla */
#endif

9
dom/media/systemservices/moz.build
View file

@ -1,4 +1,4 @@
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
@ -10,6 +10,8 @@ if CONFIG['MOZ_WEBRTC']:
'LoadManager.h',
'LoadManagerFactory.h',
'LoadMonitor.h',
'VideoEngine.h',
'VideoFrameUtils.h'
]
SOURCES += [
'CamerasChild.cpp',
@ -18,6 +20,8 @@ if CONFIG['MOZ_WEBRTC']:
'LoadManagerFactory.cpp',
'LoadMonitor.cpp',
'ShmemPool.cpp',
'VideoEngine.cpp',
'VideoFrameUtils.cpp'
]
LOCAL_INCLUDES += [
'/media/webrtc/signaling',
@ -29,6 +33,9 @@ if CONFIG['OS_TARGET'] == 'WINNT':
else:
DEFINES['WEBRTC_POSIX'] = True
if CONFIG['OS_TARGET'] == 'Android':
DEFINES['WEBRTC_ANDROID'] = True
if CONFIG['OS_TARGET'] == 'Android':
EXPORTS += [

5
dom/media/tests/mochitest/mochitest.ini
View file

@ -42,7 +42,6 @@ skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulato
skip-if = true # needed by test_enumerateDevices.html on builders
[test_ondevicechange.html]
skip-if = os == 'android'
[test_getUserMedia_active_autoplay.html]
[test_getUserMedia_audioCapture.html]
skip-if = android_version == '18' # android(Bug 1189784, timeouts on 4.3 emulator)
[test_getUserMedia_addTrackRemoveTrack.html]
@ -99,7 +98,7 @@ skip-if = toolkit == 'android' # websockets don't work on android (bug 1266217)
[test_peerConnection_basicAudioNATRelayTCP.html]
skip-if = toolkit == 'android' # websockets don't work on android (bug 1266217)
[test_peerConnection_basicAudioRequireEOC.html]
skip-if = (android_version == '18' && debug) # android(Bug 1189784, timeouts on 4.3 emulator)
skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator)
[test_peerConnection_basicAudioPcmaPcmuOnly.html]
skip-if = android_version == '18'
[test_peerConnection_basicAudioDynamicPtMissingRtpmap.html]
@ -127,7 +126,7 @@ skip-if = os == 'android' # bug 1043403
[test_peerConnection_bug822674.html]
[test_peerConnection_bug825703.html]
[test_peerConnection_bug827843.html]
skip-if = (android_version == '18' && debug) # android(Bug 1189784, timeouts on 4.3 emulator)
skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator)
[test_peerConnection_bug834153.html]
[test_peerConnection_bug1013809.html]
skip-if = (android_version == '18') # android(Bug 1189784, timeouts on 4.3 emulator)

69
dom/media/tests/mochitest/pc.js
View file

@ -1415,13 +1415,17 @@ PeerConnectionWrapper.prototype = {
* A promise that resolves when media is flowing.
*/
waitForRtpFlow(track) {
var hasFlow = stats => {
var rtp = stats.get([...stats.keys()].find(key =>
var hasFlow = (stats, retries) => {
info("Checking for stats in " + JSON.stringify(stats) + " for " + track.kind
+ " track " + track.id + ", retry number " + retries);
var rtp = stats.get([...Object.keys(stats)].find(key =>
!stats.get(key).isRemote && stats.get(key).type.endsWith("boundrtp")));
ok(rtp, "Should have RTP stats for track " + track.id);
if (!rtp) {
return false;
}
info("Should have RTP stats for track " + track.id);
info("RTP stats: "+JSON.stringify(rtp));
var nrPackets = rtp[rtp.type == "outboundrtp" ? "packetsSent"
: "packetsReceived"];
info("Track " + track.id + " has " + nrPackets + " " +
@ -1429,12 +1433,44 @@ PeerConnectionWrapper.prototype = {
return nrPackets > 0;
};
info("Checking RTP packet flow for track " + track.id);
// Time between stats checks
var retryInterval = 500;
// Timeout in ms
var timeoutInterval = 30000;
// Check hasFlow at a reasonable interval
var checkStats = new Promise((resolve, reject)=>{
var retries = 0;
var timer = setInterval(()=>{
this._pc.getStats(track).then(stats=>{
if (hasFlow(stats, retries)) {
clearInterval(timer);
ok(true, "RTP flowing for " + track.kind + " track " + track.id);
resolve();
}
retries = retries + 1;
// This is not accurate but it will tear down
// the timer eventually and probably not
// before timeoutInterval has elapsed.
if ((retries * retryInterval) > timeoutInterval) {
clearInterval(timer);
}
});
}, retryInterval);
});
var retry = (delay) => this._pc.getStats(track)
.then(stats => hasFlow(stats)? ok(true, "RTP flowing for track " + track.id) :
wait(delay).then(retry(1000)));
return retry(200);
info("Checking RTP packet flow for track " + track.id);
var retry = Promise.race([checkStats.then(new Promise((resolve, reject)=>{
info("checkStats completed for " + track.kind + " track " + track.id);
resolve();
})),
new Promise((accept,reject)=>wait(timeoutInterval).then(()=>{
info("Timeout checking for stats for track " + track.id + " after " + timeoutInterval + "ms");
reject("Timeout checking for stats for " + track.kind
+ " track " + track.id + " after " + timeoutInterval + "ms");
})
)]);
return retry;
},
/**
@ -1536,7 +1572,9 @@ PeerConnectionWrapper.prototype = {
var minimum = this.whenCreated - 1000; // on Windows XP (Bug 979649)
if (isWinXP) {
todo(false, "Can't reliably test rtcp timestamps on WinXP (Bug 979649)");
} else if (!twoMachines) {
} else if (false) { // Bug 1325430 - timestamps aren't working properly in update 49
// else if (!twoMachines) {
// Bug 1225729: On android, sometimes the first RTCP of the first
// test run gets this value, likely because no RTP has been sent yet.
if (res.timestamp != 2085978496000) {
@ -1584,8 +1622,17 @@ PeerConnectionWrapper.prototype = {
ok(rem.packetsReceived !== undefined, "Rtcp packetsReceived");
ok(rem.packetsLost !== undefined, "Rtcp packetsLost");
ok(rem.bytesReceived >= rem.packetsReceived, "Rtcp bytesReceived");
if (!this.disableRtpCountChecking) {
ok(rem.packetsReceived <= res.packetsSent, "No more than sent packets");
if (false) { // Bug 1325430 if (!this.disableRtpCountChecking) {
// no guarantee which one is newer!
// Note: this must change when we add a timestamp field to remote RTCP reports
// and make rem.timestamp be the reception time
if (res.timestamp >= rem.timestamp) {
ok(rem.packetsReceived <= res.packetsSent, "No more than sent packets");
} else {
info("REVERSED timestamps: rec:" +
rem.packetsReceived + " time:" + rem.timestamp + " sent:" + res.packetsSent + " time:" + res.timestamp);
}
// Else we may have received more than outdated Rtcp packetsSent
ok(rem.bytesReceived <= res.bytesSent, "No more than sent bytes");
}
ok(rem.jitter !== undefined, "Rtcp jitter");

12
dom/media/tests/mochitest/test_peerConnection_captureStream_canvas_2d.html
View file

@ -46,6 +46,18 @@ runNetworkTest(() => {
// i.e., this order of `requestFrame(); draw();` should work.
stream.requestFrame();
h.drawColor(canvas, h.red);
var i = 0;
return setInterval(function() {
try {
info("draw " + i ? "green" : "red");
h.drawColor(canvas, i ? h.green : h.red);
i = 1 - i;
stream.requestFrame();
} catch (e) {
// ignore; stream might have shut down, and we don't bother clearing
// the setInterval.
}
}, 500);
},
function PC_REMOTE_WAIT_FOR_REMOTE_RED() {
return h.waitForPixelColor(mediaElement, h.red, 128,

12
dom/media/tests/mochitest/test_peerConnection_captureStream_canvas_webgl.html
View file

@ -52,7 +52,7 @@ runNetworkTest(() => {
if (!program) {
ok(false, "Program should link");
return Promise.reject();
return Promise.reject("Program should link");
}
gl.useProgram(program);
@ -100,6 +100,16 @@ runNetworkTest(() => {
},
function DRAW_LOCAL_RED() {
h.drawColor(canvas, h.red);
return setInterval(function() {
try {
info("draw");
h.drawColor(canvas, h.red);
test.pcLocal.canvasStream.requestFrame();
} catch (e) {
// ignore; stream might have shut down, and we don't bother clearing
// the setInterval.
}
}, 500);
},
function WAIT_FOR_REMOTE_RED() {
return h.waitForPixelColor(vremote, h.red, 128,

27
dom/media/tests/mochitest/test_peerConnection_multiple_captureStream_canvas_2d.html
View file

@ -72,15 +72,38 @@ runNetworkTest(() => {
// After requesting a frame it will be captured at the time of next render.
// Next render will happen at next stable state, at the earliest,
// i.e., this order of `requestFrame(); draw();` should work.
stream1.requestFrame();
h.drawColor(canvas1, h.red);
stream1.requestFrame();
var i = 0;
return setInterval(function() {
try {
info("draw " + i ? "green" : "red");
h.drawColor(canvas1, i ? h.green : h.red);
i = 1 - i;
stream1.requestFrame();
} catch (e) {
// ignore; stream might have shut down, and we don't bother clearing
// the setInterval.
}
}, 500);
},
function DRAW_LOCAL2_RED() {
// After requesting a frame it will be captured at the time of next render.
// Next render will happen at next stable state, at the earliest,
// i.e., this order of `requestFrame(); draw();` should work.
stream2.requestFrame();
h.drawColor(canvas2, h.red);
stream2.requestFrame();
return setInterval(function() {
try {
info("draw");
h.drawColor(canvas2, i ? h.green : h.red);
i = 1 - i;
stream2.requestFrame();
} catch (e) {
// ignore; stream might have shut down, and we don't bother clearing
// the setInterval.
}
}, 500);
},
function WAIT_FOR_REMOTE1_RED() {
return h.waitForPixelColor(vremote1, h.red, 128,

17
dom/media/tests/mochitest/test_peerConnection_simulcastOffer.html
View file

@ -63,7 +63,8 @@
test.originalOffer.sdp, test._remote_answer.sdp);
info("Answer with RIDs: " + JSON.stringify(test._remote_answer));
ok(test._remote_answer.sdp.match(/a=simulcast:/), "Modified answer has simulcast");
ok(test._remote_answer.sdp.match(/a=rid:/), "Modified answer has rid");
ok(test._remote_answer.sdp.match(/a=rid:foo/), "Modified answer has rid foo");
ok(test._remote_answer.sdp.match(/a=rid:bar/), "Modified answer has rid bar");
}
]);
@ -118,18 +119,8 @@
ok(vremote, "Should have remote video element for pcRemote");
ok(vlocal.videoWidth > 0, "source width is positive");
ok(vlocal.videoHeight > 0, "source height is positive");
is(vremote.videoWidth, vlocal.videoWidth / 2, "sink is 1/2 width of source");
is(vremote.videoHeight, vlocal.videoHeight / 2, "sink is 1/2 height of source");
},
function PC_REMOTE_SET_RTP_NONEXISTENT_RID(test) {
// Now, cause pcRemote to filter out everything, just to make sure
// selectRecvSsrc is working.
selectRecvSsrc(test.pcRemote, 2);
},
function PC_REMOTE_ENSURE_NO_FRAMES() {
var vremote = test.pcRemote.remoteMediaElements[0];
ok(vremote, "Should have remote video element for pcRemote");
return helper.verifyNoFrames(vremote);
is(vremote.videoWidth, vlocal.videoWidth, "sink is same width as source");
is(vremote.videoHeight, vlocal.videoHeight, "sink is same height as source");
},
]);

12
dom/media/tests/mochitest/test_peerConnection_verifyVideoAfterRenegotiation.html
View file

@ -51,6 +51,18 @@ runNetworkTest(() => {
// i.e., this order of `requestFrame(); draw();` should work.
stream1.requestFrame();
h1.drawColor(canvas1, h1.red);
var i = 0;
return setInterval(function() {
try {
info("draw " + i ? "green" : "red");
h1.drawColor(canvas1, i ? h1.green : h1.red);
i = 1 - i;
stream1.requestFrame();
} catch (e) {
// ignore; stream might have shut down, and we don't bother clearing
// the setInterval.
}
}, 500);
},
function WAIT_FOR_REMOTE_RED() {
return h1.waitForPixelColor(vremote1, h1.red, 128,

18
dom/media/webrtc/MediaEngineCameraVideoSource.cpp
View file

@ -390,22 +390,4 @@ MediaEngineCameraVideoSource::SetDirectListeners(bool aHasDirectListeners)
mHasDirectListeners = aHasDirectListeners;
}
bool operator == (const webrtc::CaptureCapability& a,
const webrtc::CaptureCapability& b)
{
return a.width == b.width &&
a.height == b.height &&
a.maxFPS == b.maxFPS &&
a.rawType == b.rawType &&
a.codecType == b.codecType &&
a.expectedCaptureDelay == b.expectedCaptureDelay &&
a.interlaced == b.interlaced;
};
bool operator != (const webrtc::CaptureCapability& a,
const webrtc::CaptureCapability& b)
{
return !(a == b);
}
} // namespace mozilla

15
dom/media/webrtc/MediaEngineCameraVideoSource.h
View file

@ -11,15 +11,18 @@
// conflicts with #include of scoped_ptr.h
#undef FF
#include "webrtc/video_engine/include/vie_capture.h"
// Avoid warnings about redefinition of WARN_UNUSED_RESULT
#include "ipc/IPCMessageUtils.h"
// WebRTC includes
#include "webrtc/modules/video_capture/video_capture_defines.h"
namespace webrtc {
using CaptureCapability = VideoCaptureCapability;
}
namespace mozilla {
bool operator == (const webrtc::CaptureCapability& a,
const webrtc::CaptureCapability& b);
bool operator != (const webrtc::CaptureCapability& a,
const webrtc::CaptureCapability& b);
class MediaEngineCameraVideoSource : public MediaEngineVideoSource
{
public:

39
dom/media/webrtc/MediaEngineRemoteVideoSource.cpp
View file

@ -350,23 +350,23 @@ MediaEngineRemoteVideoSource::NotifyPull(MediaStreamGraph* aGraph,
}
}
int
MediaEngineRemoteVideoSource::FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams)
void
MediaEngineRemoteVideoSource::FrameSizeChange(unsigned int w, unsigned int h)
{
mWidth = w;
mHeight = h;
LOG(("MediaEngineRemoteVideoSource Video FrameSizeChange: %ux%u", w, h));
return 0;
#if defined(MOZ_WIDGET_GONK)
mMonitor.AssertCurrentThreadOwns(); // mWidth and mHeight are protected...
#endif
if ((mWidth < 0) || (mHeight < 0) ||
(w != (unsigned int) mWidth) || (h != (unsigned int) mHeight)) {
LOG(("MediaEngineRemoteVideoSource Video FrameSizeChange: %ux%u was %ux%u", w, h, mWidth, mHeight));
mWidth = w;
mHeight = h;
}
}
int
MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer,
size_t size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle)
MediaEngineRemoteVideoSource::DeliverFrame(uint8_t* aBuffer ,
const camera::VideoFrameProperties& aProps)
{
// Check for proper state.
if (mState != kStarted) {
@ -374,15 +374,13 @@ MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer,
return 0;
}
if ((size_t) (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2))) != size) {
MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!");
return 0;
}
// Update the dimensions
FrameSizeChange(aProps.width(), aProps.height());
// Create a video frame and append it to the track.
RefPtr<layers::PlanarYCbCrImage> image = mImageContainer->CreatePlanarYCbCrImage();
uint8_t* frame = static_cast<uint8_t*> (buffer);
uint8_t* frame = static_cast<uint8_t*> (aBuffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
@ -407,8 +405,9 @@ MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer,
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %" PRIu64 ", render_time %" PRIu64,
frame_num++, mWidth, mHeight, time_stamp, ntp_time, render_time));
LOGFRAME(("frame %d (%dx%d); timeStamp %u, ntpTimeMs %" PRIu64 ", renderTimeMs %" PRIu64,
frame_num++, mWidth, mHeight,
aProps.timeStamp(), aProps.ntpTimeMs(), aProps.renderTimeMs()));
#endif
// we don't touch anything in 'this' until here (except for snapshot,

27
dom/media/webrtc/MediaEngineRemoteVideoSource.h
View file

@ -19,6 +19,8 @@
#include "nsDirectoryServiceDefs.h"
#include "nsComponentManagerUtils.h"
// Avoid warnings about redefinition of WARN_UNUSED_RESULT
#include "ipc/IPCMessageUtils.h"
#include "VideoUtils.h"
#include "MediaEngineCameraVideoSource.h"
#include "VideoSegment.h"
@ -31,40 +33,29 @@
// WebRTC library includes follow
#include "webrtc/common.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "webrtc/video_engine/include/vie_render.h"
// Camera Access via IPC
#include "CamerasChild.h"
#include "NullTransport.h"
namespace webrtc {
class I420VideoFrame;
}
namespace mozilla {
/**
* The WebRTC implementation of the MediaEngine interface.
*/
class MediaEngineRemoteVideoSource : public MediaEngineCameraVideoSource,
public webrtc::ExternalRenderer
public camera::FrameRelay
{
typedef MediaEngineCameraVideoSource Super;
public:
NS_DECL_THREADSAFE_ISUPPORTS
// Old ExternalRenderer
void FrameSizeChange(unsigned int w, unsigned int h) override;
// ExternalRenderer
int FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams) override;
int DeliverFrame(unsigned char* buffer,
size_t size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle) override;
// XXX!!!! FIX THIS
int DeliverI420Frame(const webrtc::I420VideoFrame& webrtc_frame) override { return 0; };
bool IsTextureSupported() override { return false; };
int DeliverFrame(uint8_t* buffer,
const camera::VideoFrameProperties& properties) override;
// MediaEngineCameraVideoSource
MediaEngineRemoteVideoSource(int aIndex, mozilla::camera::CaptureEngine aCapEngine,

25
dom/media/webrtc/MediaEngineWebRTC.h
View file

@ -24,6 +24,7 @@
#include "nsComponentManagerUtils.h"
#include "nsRefPtrHashtable.h"
#include "ipc/IPCMessageUtils.h"
#include "VideoUtils.h"
#include "MediaEngineCameraVideoSource.h"
#include "VideoSegment.h"
@ -36,6 +37,8 @@
#include "MediaEngineWrapper.h"
#include "mozilla/dom/MediaStreamTrackBinding.h"
#include "CamerasChild.h"
// WebRTC library includes follow
#include "webrtc/common.h"
// Audio Engine
@ -52,11 +55,9 @@
// Video Engine
// conflicts with #include of scoped_ptr.h
#undef FF
#include "webrtc/video_engine/include/vie_base.h"
#include "webrtc/video_engine/include/vie_codec.h"
#include "webrtc/video_engine/include/vie_render.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "CamerasChild.h"
// WebRTC imports
#include "webrtc/modules/video_capture/video_capture_defines.h"
#include "NullTransport.h"
#include "AudioOutputObserver.h"
@ -256,10 +257,10 @@ public:
MOZ_ASSERT(mDevices);
if (mInUseCount == 0) {
ScopedCustomReleasePtr<webrtc::VoEExternalMedia> ptrVoERender;
ptrVoERender = webrtc::VoEExternalMedia::GetInterface(mVoiceEngine);
if (ptrVoERender) {
ptrVoERender->SetExternalRecordingStatus(true);
ScopedCustomReleasePtr<webrtc::VoEExternalMedia> ptrVoEXMedia;
ptrVoEXMedia = webrtc::VoEExternalMedia::GetInterface(mVoiceEngine);
if (ptrVoEXMedia) {
ptrVoEXMedia->SetExternalRecordingStatus(true);
}
mAnyInUse = true;
}
@ -475,9 +476,9 @@ public:
const nsString& aDeviceId) const override;
// VoEMediaProcess.
void Process(int channel, webrtc::ProcessingTypes type,
int16_t audio10ms[], int length,
int samplingFreq, bool isStereo) override;
virtual void Process(int channel, webrtc::ProcessingTypes type,
int16_t audio10ms[], size_t length,
int samplingFreq, bool isStereo) override;
void Shutdown() override;

2
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
View file

@ -824,7 +824,7 @@ typedef int16_t sample;
void
MediaEngineWebRTCMicrophoneSource::Process(int channel,
webrtc::ProcessingTypes type,
sample *audio10ms, int length,
sample *audio10ms, size_t length,
int samplingFreq, bool isStereo)
{
MOZ_ASSERT(!PassThrough(), "This should be bypassed when in PassThrough mode.");

3
dom/media/webrtc/MediaTrackConstraints.cpp
View file

@ -13,6 +13,9 @@
namespace mozilla {
using dom::ConstrainBooleanParameters;
using dom::OwningLongOrConstrainLongRange;
template<class ValueType>
template<class ConstrainRange>
void