int MediaBridgeSession::SendVideo()
{
VideoDecoder *decoder = VideoCodecFactory::CreateDecoder(VideoCodec::SORENSON);
VideoEncoder *encoder = VideoCodecFactory::CreateEncoder(rtpVideoCodec);
DWORD width = 0;
DWORD height = 0;
DWORD numpixels = 0;
QWORD lastVideoTs = 0;
Log(">SendVideo\n");
//Set video format
if (!rtpVideo.SetSendingCodec(rtpVideoCodec))
//Error
return Error("Peer do not support [%d,%s]\n",rtpVideoCodec,VideoCodec::GetNameFor(rtpVideoCodec));
//While sending video
while (sendingVideo)
{
//Wait for next video
if (!videoFrames.Wait(0))
//Check again
continue;
//Get audio grame
RTMPVideoFrame* video = videoFrames.Pop();
//check
if (!video)
//Again
continue;
//Get time difference
DWORD diff = 0;
//Get timestam
QWORD ts = video->GetTimestamp();
//If it is not the first frame
if (lastVideoTs)
//Calculate it
diff = ts - lastVideoTs;
//Set the last audio timestamp
lastVideoTs = ts;
//Check
if (video->GetVideoCodec()!=RTMPVideoFrame::FLV1)
//Error
continue;
//Decode frame
if (!decoder->Decode(video->GetMediaData(),video->GetMediaSize()))
{
Error("decode packet error");
//Next
continue;
}
//Check size
if (decoder->GetWidth()!=width || decoder->GetHeight()!=height)
{
//Get dimension
width = decoder->GetWidth();
height = decoder->GetHeight();
//Set size
numpixels = width*height*3/2;
//Set also frame rate and bps
encoder->SetFrameRate(25,300,500);
//Set them in the encoder
encoder->SetSize(width,height);
}
//Check size
if (!numpixels)
{
Error("numpixels equals 0");
//Next
continue;
}
//Check fpu
if (sendFPU)
{
//Send it
encoder->FastPictureUpdate();
//Reset
sendFPU = false;
}
//Encode it
VideoFrame *videoFrame = encoder->EncodeFrame(decoder->GetFrame(),numpixels);
//If was failed
if (!videoFrame)
{
Log("No video frame\n");
//Next
continue;
}
//Set frame time
videoFrame->SetTimestamp(diff);
//Send it smoothly
smoother.SendFrame(videoFrame,diff);
//Delete video frame
delete(video);
}
Log("<SendVideo\n");
return 1;
}
int FLVEncoder::EncodeVideo()
{
timeval prev;
//Start
Log(">FLVEncoder encode video\n");
//Allocate media frame
RTMPVideoFrame frame(0,262143);
//Check codec
switch(videoCodec)
{
case VideoCodec::SORENSON:
//Ser Video codec
frame.SetVideoCodec(RTMPVideoFrame::FLV1);
break;
case VideoCodec::H264:
//Ser Video codec
frame.SetVideoCodec(RTMPVideoFrame::AVC);
//Set NAL type
frame.SetAVCType(RTMPVideoFrame::AVCNALU);
//No delay
frame.SetAVCTS(0);
break;
default:
return Error("-Wrong codec type %d\n",videoCodec);
}
//Create the encoder
VideoEncoder *encoder = VideoCodecFactory::CreateEncoder(videoCodec,videoProperties);
///Set frame rate
encoder->SetFrameRate(fps,bitrate,intra);
//Set dimensions
encoder->SetSize(width,height);
//Start capturing
videoInput->StartVideoCapture(width,height,fps);
//The time of the first one
gettimeofday(&prev,NULL);
//No wait for first
DWORD frameTime = 0;
Log(">FLVEncoder encode vide\n");
//Mientras tengamos que capturar
while(encodingVideo)
{
//Nos quedamos con el puntero antes de que lo cambien
BYTE* pic=videoInput->GrabFrame(frameTime);
//Ensure we are still encoding
if (!encodingVideo)
break;
//Check pic
if (!pic)
continue;
//Check if we need to send intra
if (sendFPU)
{
//Set it
encoder->FastPictureUpdate();
//Do not send anymore
sendFPU = false;
}
//Encode next frame
VideoFrame *encoded = encoder->EncodeFrame(pic,videoInput->GetBufferSize());
//Check
if (!encoded)
break;
//Check size
if (frame.GetMaxMediaSize()<encoded->GetLength())
{
//Not enougth space
Error("Not enought space to copy FLV encodec frame [frame:%d,encoded:%d",frame.GetMaxMediaSize(),encoded->GetLength());
//NExt
continue;
}
//Check
if (frameTime)
{
timespec ts;
//Lock
pthread_mutex_lock(&mutex);
//Calculate timeout
calcAbsTimeout(&ts,&prev,frameTime);
//Wait next or stopped
int canceled = !pthread_cond_timedwait(&cond,&mutex,&ts);
//Unlock
pthread_mutex_unlock(&mutex);
//Check if we have been canceled
if (canceled)
//Exit
break;
}
//Set sending time of previous frame
getUpdDifTime(&prev);
//Set timestamp
encoded->SetTimestamp(getDifTime(&first)/1000);
//Set next one
frameTime = 1000/fps;
//Set duration
encoded->SetDuration(frameTime);
//Get full frame
frame.SetVideoFrame(encoded->GetData(),encoded->GetLength());
//Set buffer size
frame.SetMediaSize(encoded->GetLength());
//Check type
if (encoded->IsIntra())
//Set type
frame.SetFrameType(RTMPVideoFrame::INTRA);
else
//Set type
frame.SetFrameType(RTMPVideoFrame::INTER);
//If we need desc but yet not have it
if (!frameDesc && encoded->IsIntra() && videoCodec==VideoCodec::H264)
{
//Create new description
AVCDescriptor desc;
//Set values
desc.SetConfigurationVersion(1);
desc.SetAVCProfileIndication(0x42);
desc.SetProfileCompatibility(0x80);
desc.SetAVCLevelIndication(0x0C);
desc.SetNALUnitLength(3);
//Get encoded data
BYTE *data = encoded->GetData();
//Get size
DWORD size = encoded->GetLength();
//get from frame
desc.AddParametersFromFrame(data,size);
//Crete desc frame
frameDesc = new RTMPVideoFrame(getDifTime(&first)/1000,desc);
//Lock
pthread_mutex_lock(&mutex);
//Send it
SendMediaFrame(frameDesc);
//unlock
pthread_mutex_unlock(&mutex);
}
//Lock
pthread_mutex_lock(&mutex);
//Set timestamp
frame.SetTimestamp(encoded->GetTimeStamp());
//Publish it
SendMediaFrame(&frame);
//For each listener
for(MediaFrameListeners::iterator it = mediaListeners.begin(); it!=mediaListeners.end(); ++it)
//Send it
(*it)->onMediaFrame(RTMPMediaStream::id,*encoded);
//unlock
pthread_mutex_unlock(&mutex);
}
Log("-FLVEncoder encode video end of loop\n");
//Stop the capture
videoInput->StopVideoCapture();
//Check
if (encoder)
//Exit
delete(encoder);
Log("<FLVEncoder encode vide\n");
//Exit
return 1;
}
void H264FrameSource::doGetNextFrame()
{
// 根据 fps, 计算等待时间
double delay = 1000.0 / videoFPS ;
int to_delay = delay * 1000; // us
if(!m_videoInput)
return;
BYTE *pic = m_videoInput->GrabFrame();
//Check picture
if (!pic) {
fFrameSize = 0;
m_started = 0;
return;
}
//Check if we need to send intra
if (sendFPU)
{
videoEncoder->FastPictureUpdate();
}
//if (fPresentationTime.tv_sec == 0 && fPresentationTime.tv_usec == 0) {
// This is the first frame, so use the current time:
//} else {
// Increment by the play time of the previous data:
// unsigned uSeconds = fPresentationTime.tv_usec + fLastPlayTime;
// fPresentationTime.tv_sec += uSeconds/1000000;
// fPresentationTime.tv_usec = uSeconds%1000000;
//}
// Remember the play time of this data:
//fLastPlayTime = (fPlayTimePerFrame*fFrameSize)/fPreferredFrameSize;
//fDurationInMicroseconds = fLastPlayTime;
//fDurationInMicroseconds = 1000.0 / videoFPS;
VideoFrame *videoFrame = videoEncoder->EncodeFrame(pic,m_videoInput->GetBufferSize());
//If was failed
if (!videoFrame){
//Next
fFrameSize = 0;
m_started = 0;
Log("-----Error encoding video\n");
double delay = 1000.0 / videoFPS;
int to_delay = delay * 1000; // us
nextTask() = envir().taskScheduler().scheduleDelayedTask(to_delay,
(TaskFunc*)FramedSource::afterGetting, this);
return;
}
if(sendFPU)
sendFPU = false;
//Set frame timestamp
videoFrame->SetTimestamp(getDifTime(&first)/1000);
//Set sending time of previous frame
//getUpdDifTime(&prev);
//gettimeofday(&fPresentationTime, 0);
fFrameSize = videoFrame->GetLength();
memmove(fTo, videoFrame->GetData(), fFrameSize);
if (fFrameSize > fMaxSize) {
fNumTruncatedBytes = fFrameSize - fMaxSize;
fFrameSize = fMaxSize;
}
else {
fNumTruncatedBytes = 0;
}
gettimeofday(&fPresentationTime, NULL);
//to_delay = ((1000 / videoFPS) * fFrameSize / RTPPAYLOADSIZE) * 1000; // us
nextTask() = envir().taskScheduler().scheduleDelayedTask(to_delay,
(TaskFunc*)FramedSource::afterGetting, this);
}
/*******************************************
* SendVideo
* Capturamos el video y lo mandamos
*******************************************/
int VideoStream::SendVideo()
{
timeval prev;
timeval lastFPU;
DWORD num = 0;
QWORD overslept = 0;
Acumulator bitrateAcu(1000);
Acumulator fpsAcu(1000);
Log(">SendVideo [width:%d,size:%d,bitrate:%d,fps:%d,intra:%d]\n",videoGrabWidth,videoGrabHeight,videoBitrate,videoFPS,videoIntraPeriod);
//Creamos el encoder
VideoEncoder* videoEncoder = VideoCodecFactory::CreateEncoder(videoCodec,videoProperties);
//Comprobamos que se haya creado correctamente
if (videoEncoder == NULL)
//error
return Error("Can't create video encoder\n");
//Comrpobamos que tengamos video de entrada
if (videoInput == NULL)
return Error("No video input");
//Iniciamos el tama�o del video
if (!videoInput->StartVideoCapture(videoGrabWidth,videoGrabHeight,videoFPS))
return Error("Couldn't set video capture\n");
//Start at 80%
int current = videoBitrate*0.8;
//Send at higher bitrate first frame, but skip frames after that so sending bitrate is kept
videoEncoder->SetFrameRate(videoFPS,current*5,videoIntraPeriod);
//No wait for first
QWORD frameTime = 0;
//Iniciamos el tamama�o del encoder
videoEncoder->SetSize(videoGrabWidth,videoGrabHeight);
//The time of the previos one
gettimeofday(&prev,NULL);
//Fist FPU
gettimeofday(&lastFPU,NULL);
//Started
Log("-Sending video\n");
//Mientras tengamos que capturar
while(sendingVideo)
{
//Nos quedamos con el puntero antes de que lo cambien
BYTE *pic = videoInput->GrabFrame(frameTime/1000);
//Check picture
if (!pic)
//Exit
continue;
//Check if we need to send intra
if (sendFPU)
{
//Do not send anymore
sendFPU = false;
//Do not send if we just send one (100ms)
if (getDifTime(&lastFPU)/1000>minFPUPeriod)
{
//Send at higher bitrate first frame, but skip frames after that so sending bitrate is kept
videoEncoder->SetFrameRate(videoFPS,current*5,videoIntraPeriod);
//Reste frametime so it is calcualted afterwards
frameTime = 0;
//Set it
videoEncoder->FastPictureUpdate();
//Update last FPU
getUpdDifTime(&lastFPU);
}
}
//Calculate target bitrate
int target = current;
//Check temporal limits for estimations
if (bitrateAcu.IsInWindow())
{
//Get real sent bitrate during last second and convert to kbits
DWORD instant = bitrateAcu.GetInstantAvg()/1000;
//If we are in quarentine
if (videoBitrateLimitCount)
//Limit sending bitrate
target = videoBitrateLimit;
//Check if sending below limits
else if (instant<videoBitrate)
//Increase a 8% each second or fps kbps
target += (DWORD)(target*0.08/videoFPS)+1;
}
//Check target bitrate agains max conf bitrate
if (target>videoBitrate*1.2)
//Set limit to max bitrate allowing a 20% overflow so instant bitrate can get closer to target
target = videoBitrate*1.2;
//Check limits counter
if (videoBitrateLimitCount>0)
//One frame less of limit
videoBitrateLimitCount--;
//Check if we have a new bitrate
if (target && target!=current)
{
//Reset bitrate
videoEncoder->SetFrameRate(videoFPS,target,videoIntraPeriod);
//Upate current
current = target;
}
//Procesamos el frame
VideoFrame *videoFrame = videoEncoder->EncodeFrame(pic,videoInput->GetBufferSize());
//If was failed
if (!videoFrame)
//Next
continue;
//Increase frame counter
fpsAcu.Update(getTime()/1000,1);
//Check
if (frameTime)
{
timespec ts;
//Lock
pthread_mutex_lock(&mutex);
//Calculate slept time
QWORD sleep = frameTime;
//Remove extra sleep from prev
if (overslept<sleep)
//Remove it
sleep -= overslept;
else
//Do not overflow
sleep = 1;
//Calculate timeout
calcAbsTimeoutNS(&ts,&prev,sleep);
//Wait next or stopped
int canceled = !pthread_cond_timedwait(&cond,&mutex,&ts);
//Unlock
pthread_mutex_unlock(&mutex);
//Check if we have been canceled
if (canceled)
//Exit
break;
//Get differencence
QWORD diff = getDifTime(&prev);
//If it is biffer
if (diff>frameTime)
//Get what we have slept more
overslept = diff-frameTime;
else
//No oversletp (shoulddn't be possible)
overslept = 0;
}
//Increase frame counter
fpsAcu.Update(getTime()/1000,1);
//If first
if (!frameTime)
{
//Set frame time, slower
frameTime = 5*1000000/videoFPS;
//Restore bitrate
videoEncoder->SetFrameRate(videoFPS,current,videoIntraPeriod);
} else {
//Set frame time
frameTime = 1000000/videoFPS;
}
//Add frame size in bits to bitrate calculator
bitrateAcu.Update(getDifTime(&ini)/1000,videoFrame->GetLength()*8);
//Set frame timestamp
videoFrame->SetTimestamp(getDifTime(&ini)/1000);
//Check if we have mediaListener
if (mediaListener)
//Call it
mediaListener->onMediaFrame(*videoFrame);
//Set sending time of previous frame
getUpdDifTime(&prev);
//Calculate sending times based on bitrate
DWORD sendingTime = videoFrame->GetLength()*8/current;
//Adjust to maximum time
if (sendingTime>frameTime/1000)
//Cap it
sendingTime = frameTime/1000;
//If it was a I frame
if (videoFrame->IsIntra())
//Clean rtp rtx buffer
rtp.FlushRTXPackets();
//Send it smoothly
smoother.SendFrame(videoFrame,sendingTime);
//Dump statistics
if (num && ((num%videoFPS*10)==0))
{
Debug("-Send bitrate target=%d current=%d avg=%llf rate=[%llf,%llf] fps=[%llf,%llf] limit=%d\n",target,current,bitrateAcu.GetInstantAvg()/1000,bitrateAcu.GetMinAvg()/1000,bitrateAcu.GetMaxAvg()/1000,fpsAcu.GetMinAvg(),fpsAcu.GetMaxAvg(),videoBitrateLimit);
bitrateAcu.ResetMinMax();
fpsAcu.ResetMinMax();
}
num++;
}
Log("-SendVideo out of loop\n");
//Terminamos de capturar
videoInput->StopVideoCapture();
//Check
if (videoEncoder)
//Borramos el encoder
delete videoEncoder;
//Salimos
Log("<SendVideo [%d]\n",sendingVideo);
return 0;
}
QWORD MP4RtpTrack::Read(Listener *listener)
{
int last = 0;
uint8_t* data;
bool isSyncSample;
// If it's first packet of a frame
if (!numHintSamples)
{
// Get number of rtp packets for this sample
if (!MP4ReadRtpHint(mp4, hint, sampleId, &numHintSamples))
{
//Print error
Error("Error reading hintt");
//Exit
return MP4_INVALID_TIMESTAMP;
}
// Get number of samples for this sample
frameSamples = MP4GetSampleDuration(mp4, hint, sampleId);
// Get size of sample
frameSize = MP4GetSampleSize(mp4, hint, sampleId);
// Get sample timestamp
frameTime = MP4GetSampleTime(mp4, hint, sampleId);
//Convert to miliseconds
frameTime = MP4ConvertFromTrackTimestamp(mp4, hint, frameTime, 1000);
// Check if it is H264 and it is a Sync frame
if (codec==VideoCodec::H264 && MP4GetSampleSync(mp4,track,sampleId))
// Send SEI info
SendH263SEI(listener);
//Get max data lenght
BYTE *data = NULL;
DWORD dataLen = 0;
MP4Timestamp startTime;
MP4Duration duration;
MP4Duration renderingOffset;
//Get values
data = frame->GetData();
dataLen = frame->GetMaxMediaLength();
// Read next rtp packet
if (!MP4ReadSample(
mp4, // MP4FileHandle hFile
track, // MP4TrackId hintTrackId
sampleId, // MP4SampleId sampleId,
(u_int8_t **) &data, // u_int8_t** ppBytes
(u_int32_t *) &dataLen, // u_int32_t* pNumBytes
&startTime, // MP4Timestamp* pStartTime
&duration, // MP4Duration* pDuration
&renderingOffset, // MP4Duration* pRenderingOffset
&isSyncSample // bool* pIsSyncSample
))
{
Error("Error reading sample");
//Last
return MP4_INVALID_TIMESTAMP;
}
//Check type
if (media == MediaFrame::Video)
{
//Get video frame
VideoFrame *video = (VideoFrame*)frame;
//Set lenght
video->SetLength(dataLen);
//Timestamp
video->SetTimestamp(startTime*90000/timeScale);
//Set intra
video->SetIntra(isSyncSample);
} else {
//Get Audio frame
AudioFrame *audio = (AudioFrame*)frame;
//Set lenght
audio->SetLength(dataLen);
//Timestamp
audio->SetTimestamp(startTime*8000/timeScale);
}
//Check listener
if (listener)
//Frame callback
listener->onMediaFrame(*frame);
}
// if it's the last
if (packetIndex + 1 == numHintSamples)
//Set last mark
last = 1;
// Set mark bit
rtp.SetMark(last);
// Get data pointer
data = rtp.GetMediaData();
//Get max data lenght
DWORD dataLen = rtp.GetMaxMediaLength();
// Read next rtp packet
if (!MP4ReadRtpPacket(
mp4, // MP4FileHandle hFile
hint, // MP4TrackId hintTrackId
packetIndex++, // u_int16_t packetIndex
(u_int8_t **) &data, // u_int8_t** ppBytes
(u_int32_t *) &dataLen, // u_int32_t* pNumBytes
0, // u_int32_t ssrc DEFAULT(0)
0, // bool includeHeader DEFAULT(true)
1 // bool includePayload DEFAULT(true)
))
{
//Error
Error("Error reading packet [%d,%d,%d]\n", hint, track,packetIndex);
//Exit
return MP4_INVALID_TIMESTAMP;
}
//Check
if (dataLen>rtp.GetMaxMediaLength())
{
//Error
Error("RTP packet too big [%u,%u]\n",dataLen,rtp.GetMaxMediaLength());
//Exit
return MP4_INVALID_TIMESTAMP;
}
//Set lenght
rtp.SetMediaLength(dataLen);
// Write frame
listener->onRTPPacket(rtp);
// Are we the last packet in a hint?
if (last)
{
// The first hint
packetIndex = 0;
// Go for next sample
sampleId++;
numHintSamples = 0;
//Return next frame time
return GetNextFrameTime();
}
// This packet is this one
return frameTime;
}
