// Called if the graph thinks it's running out of buffered video; repeat
// the last frame for whatever minimum period it think it needs. Note that
// this means that no *real* frame can be inserted during this period.
void
MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aID,
StreamTime aDesiredTime)
{
VideoSegment segment;
MonitorAutoLock lock(mMonitor);
// B2G does AddTrack, but holds kStarted until the hardware changes state.
// So mState could be kReleased here. We really don't care about the state,
// though.
StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID);
LOGFRAME(("NotifyPull, desired = %ld, delta = %ld %s", (int64_t) aDesiredTime,
(int64_t) delta, mImage.get() ? "" : "<null>"));
// Bug 846188 We may want to limit incoming frames to the requested frame rate
// mFps - if you want 30FPS, and the camera gives you 60FPS, this could
// cause issues.
// We may want to signal if the actual frame rate is below mMinFPS -
// cameras often don't return the requested frame rate especially in low
// light; we should consider surfacing this so that we can switch to a
// lower resolution (which may up the frame rate)
// Don't append if we've already provided a frame that supposedly goes past the current aDesiredTime
// Doing so means a negative delta and thus messes up handling of the graph
if (delta > 0) {
// nullptr images are allowed
AppendToTrack(aSource, mImage, aID, delta);
}
}
void
MediaEngineRemoteVideoSource::NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aID, StreamTime aDesiredTime)
{
VideoSegment segment;
MonitorAutoLock lock(mMonitor);
StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID);
if (delta > 0) {
// nullptr images are allowed
AppendToTrack(aSource, mImage, aID, delta);
}
}
nsresult
MediaEngineGonkVideoSource::OnNewMediaBufferFrame(MediaBuffer* aBuffer)
{
{
ReentrantMonitorAutoEnter sync(mCallbackMonitor);
if (mState == kStopped) {
return NS_OK;
}
}
MonitorAutoLock enter(mMonitor);
if (mImage) {
if (mImage->AsGrallocImage()) {
// MediaEngineGonkVideoSource expects that GrallocImage is GonkCameraImage.
// See Bug 938034.
GonkCameraImage* cameraImage = static_cast<GonkCameraImage*>(mImage.get());
cameraImage->SetMediaBuffer(aBuffer);
} else {
LOG(("mImage is non-GrallocImage"));
}
uint32_t len = mSources.Length();
for (uint32_t i = 0; i < len; i++) {
if (mSources[i]) {
// Duration is 1 here.
// Ideally, it should be camera timestamp here and the MSG will have
// enough sample duration without calling NotifyPull() anymore.
// Unfortunately, clock in gonk camera looks like is a different one
// comparing to MSG. As result, it causes time inaccurate. (frames be
// queued in MSG longer and longer as time going by in device like Frame)
AppendToTrack(mSources[i], mImage, mTrackID, 1);
}
}
if (mImage->AsGrallocImage()) {
GonkCameraImage* cameraImage = static_cast<GonkCameraImage*>(mImage.get());
// Clear MediaBuffer immediately, it prevents MediaBuffer is kept in
// MediaStreamGraph thread.
cameraImage->ClearMediaBuffer();
}
}
return NS_OK;
}
void
MediaEngineRemoteVideoSource::NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aID, StreamTime aDesiredTime,
const PrincipalHandle& aPrincipalHandle)
{
VideoSegment segment;
MonitorAutoLock lock(mMonitor);
if (mState != kStarted) {
return;
}
StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID);
if (delta > 0) {
// nullptr images are allowed
AppendToTrack(aSource, mImage, aID, delta, aPrincipalHandle);
}
}
// ViEExternalRenderer Callback. Process every incoming frame here.
int
MediaEngineWebRTCVideoSource::DeliverFrame(
unsigned char* buffer, int size, uint32_t time_stamp,
int64_t ntp_time_ms, int64_t render_time, void *handle)
{
// Check for proper state.
if (mState != kStarted) {
LOG(("DeliverFrame: video not started"));
return 0;
}
if (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2)) != size) {
MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!");
return 0;
}
// Create a video frame and append it to the track.
nsRefPtr<layers::Image> image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR);
layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
uint8_t* frame = static_cast<uint8_t*> (buffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
// Take lots of care to round up!
layers::PlanarYCbCrData data;
data.mYChannel = frame;
data.mYSize = IntSize(mWidth, mHeight);
data.mYStride = (mWidth * lumaBpp + 7)/ 8;
data.mCbCrStride = (mWidth * chromaBpp + 7) / 8;
data.mCbChannel = frame + mHeight * data.mYStride;
data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride;
data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = IntSize(mWidth, mHeight);
data.mStereoMode = StereoMode::MONO;
videoImage->SetData(data);
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %lu, render_time %lu", frame_num++,
mWidth, mHeight, time_stamp, ntp_time_ms, render_time));
#endif
// we don't touch anything in 'this' until here (except for snapshot,
// which has it's own lock)
MonitorAutoLock lock(mMonitor);
// implicitly releases last image
mImage = image.forget();
// Push the frame into the MSG with a minimal duration. This will likely
// mean we'll still get NotifyPull calls which will then return the same
// frame again with a longer duration. However, this means we won't
// fail to get the frame in and drop frames.
// XXX The timestamp for the frame should be based on the Capture time,
// not the MSG time, and MSG should never, ever block on a (realtime)
// video frame (or even really for streaming - audio yes, video probably no).
// Note that MediaPipeline currently ignores the timestamps from MSG
uint32_t len = mSources.Length();
for (uint32_t i = 0; i < len; i++) {
if (mSources[i]) {
AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration
}
}
return 0;
}
