// Called if the graph thinks it's running out of buffered video; repeat
// the last frame for whatever minimum period it think it needs. Note that
// this means that no *real* frame can be inserted during this period.
void
MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aID,
StreamTime aDesiredTime)
{
VideoSegment segment;
MonitorAutoLock lock(mMonitor);
// B2G does AddTrack, but holds kStarted until the hardware changes state.
// So mState could be kReleased here. We really don't care about the state,
// though.
StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID);
LOGFRAME(("NotifyPull, desired = %ld, delta = %ld %s", (int64_t) aDesiredTime,
(int64_t) delta, mImage.get() ? "" : "<null>"));
// Bug 846188 We may want to limit incoming frames to the requested frame rate
// mFps - if you want 30FPS, and the camera gives you 60FPS, this could
// cause issues.
// We may want to signal if the actual frame rate is below mMinFPS -
// cameras often don't return the requested frame rate especially in low
// light; we should consider surfacing this so that we can switch to a
// lower resolution (which may up the frame rate)
// Don't append if we've already provided a frame that supposedly goes past the current aDesiredTime
// Doing so means a negative delta and thus messes up handling of the graph
if (delta > 0) {
// nullptr images are allowed
AppendToTrack(aSource, mImage, aID, delta);
}
}
int
MediaEngineRemoteVideoSource::DeliverFrame(uint8_t* aBuffer ,
const camera::VideoFrameProperties& aProps)
{
// Check for proper state.
if (mState != kStarted) {
LOG(("DeliverFrame: video not started"));
return 0;
}
// Update the dimensions
FrameSizeChange(aProps.width(), aProps.height());
// Create a video frame and append it to the track.
RefPtr<layers::PlanarYCbCrImage> image = mImageContainer->CreatePlanarYCbCrImage();
uint8_t* frame = static_cast<uint8_t*> (aBuffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
// Take lots of care to round up!
layers::PlanarYCbCrData data;
data.mYChannel = frame;
data.mYSize = IntSize(mWidth, mHeight);
data.mYStride = (mWidth * lumaBpp + 7)/ 8;
data.mCbCrStride = (mWidth * chromaBpp + 7) / 8;
data.mCbChannel = frame + mHeight * data.mYStride;
data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride;
data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = IntSize(mWidth, mHeight);
data.mStereoMode = StereoMode::MONO;
if (!image->CopyData(data)) {
MOZ_ASSERT(false);
return 0;
}
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timeStamp %u, ntpTimeMs %" PRIu64 ", renderTimeMs %" PRIu64,
frame_num++, mWidth, mHeight,
aProps.timeStamp(), aProps.ntpTimeMs(), aProps.renderTimeMs()));
#endif
// we don't touch anything in 'this' until here (except for snapshot,
// which has it's own lock)
MonitorAutoLock lock(mMonitor);
// implicitly releases last image
mImage = image.forget();
// We'll push the frame into the MSG on the next NotifyPull. This will avoid
// swamping the MSG with frames should it be taking longer than normal to run
// an iteration.
return 0;
}
// ViEExternalRenderer Callback. Process every incoming frame here.
int
MediaEngineWebRTCVideoSource::DeliverFrame(
unsigned char* buffer, int size, uint32_t time_stamp, int64_t render_time,
void *handle)
{
// Check for proper state.
if (mState != kStarted) {
LOG(("DeliverFrame: video not started"));
return 0;
}
if (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2)) != size) {
MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!");
return 0;
}
// Create a video frame and append it to the track.
nsRefPtr<layers::Image> image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR);
layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
uint8_t* frame = static_cast<uint8_t*> (buffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
// Take lots of care to round up!
layers::PlanarYCbCrData data;
data.mYChannel = frame;
data.mYSize = IntSize(mWidth, mHeight);
data.mYStride = (mWidth * lumaBpp + 7)/ 8;
data.mCbCrStride = (mWidth * chromaBpp + 7) / 8;
data.mCbChannel = frame + mHeight * data.mYStride;
data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride;
data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = IntSize(mWidth, mHeight);
data.mStereoMode = StereoMode::MONO;
videoImage->SetData(data);
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timestamp %u, render_time %lu", frame_num++,
mWidth, mHeight, time_stamp, render_time));
#endif
// we don't touch anything in 'this' until here (except for snapshot,
// which has it's own lock)
MonitorAutoLock lock(mMonitor);
// implicitly releases last image
mImage = image.forget();
return 0;
}
// Called if the graph thinks it's running out of buffered video; repeat
// the last frame for whatever minimum period it think it needs. Note that
// this means that no *real* frame can be inserted during this period.
void
MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream *aSource,
TrackID aID,
StreamTime aDesiredTime,
TrackTicks &aLastEndTime)
{
VideoSegment segment;
MonitorAutoLock lock(mMonitor);
if (mState != kStarted)
return;
// Note: we're not giving up mImage here
nsRefPtr<layers::Image> image = mImage;
TrackTicks target = TimeToTicksRoundUp(USECS_PER_S, aDesiredTime);
TrackTicks delta = target - aLastEndTime;
LOGFRAME(("NotifyPull, desired = %ld, target = %ld, delta = %ld %s", (int64_t) aDesiredTime,
(int64_t) target, (int64_t) delta, image ? "" : "<null>"));
// Bug 846188 We may want to limit incoming frames to the requested frame rate
// mFps - if you want 30FPS, and the camera gives you 60FPS, this could
// cause issues.
// We may want to signal if the actual frame rate is below mMinFPS -
// cameras often don't return the requested frame rate especially in low
// light; we should consider surfacing this so that we can switch to a
// lower resolution (which may up the frame rate)
// Don't append if we've already provided a frame that supposedly goes past the current aDesiredTime
// Doing so means a negative delta and thus messes up handling of the graph
if (delta > 0) {
// nullptr images are allowed
if (image) {
segment.AppendFrame(image.forget(), delta, gfxIntSize(mWidth, mHeight));
} else {
segment.AppendFrame(nullptr, delta, gfxIntSize(0,0));
}
// This can fail if either a) we haven't added the track yet, or b)
// we've removed or finished the track.
if (aSource->AppendToTrack(aID, &(segment))) {
aLastEndTime = target;
}
}
}
// ViEExternalRenderer Callback. Process every incoming frame here.
int
MediaEngineWebRTCVideoSource::DeliverFrame(
unsigned char* buffer, int size, uint32_t time_stamp, int64_t render_time,
void *handle)
{
// mInSnapshotMode can only be set before the camera is turned on and
// the renderer is started, so this amounts to a 1-shot
if (mInSnapshotMode) {
// Set the condition variable to false and notify Snapshot().
MonitorAutoLock lock(mMonitor);
mInSnapshotMode = false;
lock.Notify();
return 0;
}
// Check for proper state.
if (mState != kStarted) {
LOG(("DeliverFrame: video not started"));
return 0;
}
MOZ_ASSERT(mWidth*mHeight*3/2 == size);
if (mWidth*mHeight*3/2 != size) {
return 0;
}
// Create a video frame and append it to the track.
ImageFormat format = PLANAR_YCBCR;
nsRefPtr<layers::Image> image = mImageContainer->CreateImage(&format, 1);
layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
uint8_t* frame = static_cast<uint8_t*> (buffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
layers::PlanarYCbCrData data;
data.mYChannel = frame;
data.mYSize = gfxIntSize(mWidth, mHeight);
data.mYStride = mWidth * lumaBpp/ 8;
data.mCbCrStride = mWidth * chromaBpp / 8;
data.mCbChannel = frame + mHeight * data.mYStride;
data.mCrChannel = data.mCbChannel + mHeight * data.mCbCrStride / 2;
data.mCbCrSize = gfxIntSize(mWidth/ 2, mHeight/ 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = gfxIntSize(mWidth, mHeight);
data.mStereoMode = STEREO_MODE_MONO;
videoImage->SetData(data);
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timestamp %u, render_time %lu", frame_num++,
mWidth, mHeight, time_stamp, render_time));
#endif
// we don't touch anything in 'this' until here (except for snapshot,
// which has it's own lock)
MonitorAutoLock lock(mMonitor);
// implicitly releases last image
mImage = image.forget();
return 0;
}
// ViEExternalRenderer Callback. Process every incoming frame here.
int
MediaEngineWebRTCVideoSource::DeliverFrame(
unsigned char* buffer, int size, uint32_t time_stamp,
int64_t ntp_time_ms, int64_t render_time, void *handle)
{
// Check for proper state.
if (mState != kStarted) {
LOG(("DeliverFrame: video not started"));
return 0;
}
if (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2)) != size) {
MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!");
return 0;
}
// Create a video frame and append it to the track.
nsRefPtr<layers::Image> image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR);
layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
uint8_t* frame = static_cast<uint8_t*> (buffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
// Take lots of care to round up!
layers::PlanarYCbCrData data;
data.mYChannel = frame;
data.mYSize = IntSize(mWidth, mHeight);
data.mYStride = (mWidth * lumaBpp + 7)/ 8;
data.mCbCrStride = (mWidth * chromaBpp + 7) / 8;
data.mCbChannel = frame + mHeight * data.mYStride;
data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride;
data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = IntSize(mWidth, mHeight);
data.mStereoMode = StereoMode::MONO;
videoImage->SetData(data);
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %lu, render_time %lu", frame_num++,
mWidth, mHeight, time_stamp, ntp_time_ms, render_time));
#endif
// we don't touch anything in 'this' until here (except for snapshot,
// which has it's own lock)
MonitorAutoLock lock(mMonitor);
// implicitly releases last image
mImage = image.forget();
// Push the frame into the MSG with a minimal duration. This will likely
// mean we'll still get NotifyPull calls which will then return the same
// frame again with a longer duration. However, this means we won't
// fail to get the frame in and drop frames.
// XXX The timestamp for the frame should be based on the Capture time,
// not the MSG time, and MSG should never, ever block on a (realtime)
// video frame (or even really for streaming - audio yes, video probably no).
// Note that MediaPipeline currently ignores the timestamps from MSG
uint32_t len = mSources.Length();
for (uint32_t i = 0; i < len; i++) {
if (mSources[i]) {
AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration
}
}
return 0;
}
