UserTracker::Obj::Obj( xn::Context context )
: mContext( context )
{
XnStatus rc;
rc = mContext.FindExistingNode(XN_NODE_TYPE_DEPTH, mDepthGenerator);
checkRc( rc, "Retrieving depth generator" );
xn::DepthMetaData depthMD;
mDepthGenerator.GetMetaData( depthMD );
mDepthWidth = depthMD.FullXRes();
mDepthHeight = depthMD.FullYRes();
mUserBuffers = BufferManager<uint8_t>( mDepthWidth * mDepthHeight, this );
rc = mContext.FindExistingNode(XN_NODE_TYPE_USER, mUserGenerator);
if (rc != XN_STATUS_OK)
{
rc = mUserGenerator.Create(mContext);
if (rc != XN_STATUS_OK)
throw ExcFailedUserGeneratorInit();
}
if (!mUserGenerator.IsCapabilitySupported(XN_CAPABILITY_SKELETON))
throw ExcNoSkeletonCapability();
// user callbacks
XnCallbackHandle userCallbacks;
rc = mUserGenerator.RegisterUserCallbacks(newUserCB, lostUserCB,
this, userCallbacks);
checkRc( rc, "Register user callbacks" );
// calibration callbacks
XnCallbackHandle calibrationCallbacks;
rc = mUserGenerator.GetSkeletonCap().RegisterCalibrationCallbacks(
calibrationStartCB, calibrationEndCB,
this, calibrationCallbacks);
checkRc( rc, "Register calibration callbacks" );
if (mUserGenerator.GetSkeletonCap().NeedPoseForCalibration())
{
sNeedPose = true;
if (!mUserGenerator.IsCapabilitySupported(XN_CAPABILITY_POSE_DETECTION))
throw ExcNoPoseDetection();
XnCallbackHandle poseDetected;
rc = mUserGenerator.GetPoseDetectionCap().RegisterToPoseDetected(
userPoseDetectedCB, this, poseDetected);
checkRc( rc, "Register pose detected callback" );
mUserGenerator.GetSkeletonCap().GetCalibrationPose(sCalibrationPose);
}
mUserGenerator.GetSkeletonCap().SetSkeletonProfile(XN_SKEL_PROFILE_ALL);
}
void UserTracker::Obj::stop()
{
if ( mUserGenerator.IsValid() && mUserGenerator.IsGenerating() )
{
XnStatus rc;
rc = mUserGenerator.StopGenerating();
checkRc( rc, "UserGenerator.StopGenerating" );
}
}
void MovieDecoder::getScaledVideoFrame(int scaledSize, bool maintainAspectRatio, VideoFrame& videoFrame)
{
initializeFilterGraph(m_pFormatContext->streams[m_VideoStream]->time_base, scaledSize, maintainAspectRatio);
auto del = [] (AVFrame* f) { av_frame_free(&f); };
std::unique_ptr<AVFrame, decltype(del)> res(av_frame_alloc(), del);
checkRc(av_buffersrc_write_frame(m_pFilterSource, m_pFrame), "Failed to write frame to filter graph");
int attempts = 0;
int rc = av_buffersink_get_frame(m_pFilterSink, res.get());
while (rc == AVERROR(EAGAIN) && attempts++ < 10)
{
decodeVideoFrame();
checkRc(av_buffersrc_write_frame(m_pFilterSource, m_pFrame), "Failed to write frame to filter graph");
rc = av_buffersink_get_frame(m_pFilterSink, res.get());
}
checkRc(rc, "Failed to get buffer from filter");
videoFrame.width = res->width;
videoFrame.height = res->height;
videoFrame.lineSize = videoFrame.width * 4;
if(videoFrame.frameData != nullptr)
delete videoFrame.frameData;
uint8_t * framedata = res->data[0];
videoFrame.frameData = new uint8_t[videoFrame.width * 4 * videoFrame.height];
for(int y = 0;y < videoFrame.height;y++)
{
memcpy(videoFrame.frameData + ((videoFrame.height - y - 1) * videoFrame.lineSize), framedata + (y * res->linesize[0]), videoFrame.lineSize);
}
if (m_pFilterGraph)
{
avfilter_graph_free(&m_pFilterGraph);
}
}
ImageSourceRef UserTracker::getUserMask( XnUserID userId /* = 0 */, bool fillWithUserId /* = false */ )
{
mObj->mUserBuffers.derefActiveBuffer(); // finished with current active buffer
uint8_t *destPixels = mObj->mUserBuffers.getNewBuffer(); // request a new buffer
xn::SceneMetaData sceneMD;
XnStatus rc = mObj->mUserGenerator.GetUserPixels( userId, sceneMD);
checkRc( rc, "UserGenerator GetUserPixels" );
if ( rc == XN_STATUS_OK )
{
const uint16_t *userPixels = reinterpret_cast< const uint16_t * >( sceneMD.Data() );
if ( fillWithUserId )
{
if ( userId == 0 )
{
// mask for all users
for ( size_t i = 0 ; i < mObj->mDepthWidth * mObj->mDepthHeight; i++ )
{
destPixels[i] = userPixels[i] & 255;
}
}
else
{
// mask for specified user id
for ( size_t i = 0 ; i < mObj->mDepthWidth * mObj->mDepthHeight; i++ )
{
if ( userPixels[i] == userId )
destPixels[i] = userPixels[i] & 255;
else
destPixels[i] = 0;
}
}
}
else
{
if ( userId == 0 )
{
// mask for all users
for ( size_t i = 0 ; i < mObj->mDepthWidth * mObj->mDepthHeight; i++ )
{
if ( userPixels[i] != 0 )
destPixels[i] = 255;
else
destPixels[i] = 0;
}
}
else
{
// mask for specified user id
for ( size_t i = 0 ; i < mObj->mDepthWidth * mObj->mDepthHeight; i++ )
{
if ( userPixels[i] == userId )
destPixels[i] = 255;
else
destPixels[i] = 0;
}
}
}
}
// set this new buffer to be the current active buffer
mObj->mUserBuffers.setActiveBuffer( destPixels );
uint8_t *activeMask = mObj->mUserBuffers.refActiveBuffer();
return ImageSourceRef( new ImageSourceOpenNIUserMask( activeMask, mObj->mDepthWidth, mObj->mDepthHeight, mObj ) );
}
void UserTracker::Obj::start()
{
XnStatus rc;
rc = mUserGenerator.StartGenerating();
checkRc( rc, "UserGenerator.StartGenerating" );
}
void MovieDecoder::initializeFilterGraph(const AVRational& timeBase, int size, bool maintainAspectRatio)
{
static const AVPixelFormat pixelFormats[] = { AV_PIX_FMT_BGRA, AV_PIX_FMT_NONE };
auto del = [] (AVBufferSinkParams* p) { av_freep(p); };
std::unique_ptr<AVBufferSinkParams, decltype(del)> buffersinkParams(av_buffersink_params_alloc(), del);
m_pFilterGraph = avfilter_graph_alloc();
assert(m_pFilterGraph);
std::stringstream ss;
ss << "video_size=" << GetVideoWidth() << "x" << GetVideoHeight()
<< ":pix_fmt=" << m_pVideoCodecContext->pix_fmt
<< ":time_base=" << timeBase.num << "/" << timeBase.den
<< ":pixel_aspect=" << m_pVideoCodecContext->sample_aspect_ratio.num << "/" << FFMAX(m_pVideoCodecContext->sample_aspect_ratio.den, 1);
checkRc(avfilter_graph_create_filter(&m_pFilterSource, avfilter_get_by_name("buffer"), "thumb_buffer", ss.str().c_str(), nullptr, m_pFilterGraph),
"Failed to create filter source");
buffersinkParams->pixel_fmts = pixelFormats;
checkRc(avfilter_graph_create_filter(&m_pFilterSink, avfilter_get_by_name("buffersink"), "thumb_buffersink", nullptr, buffersinkParams.get(), m_pFilterGraph),
"Failed to create filter sink");
buffersinkParams.release();
AVFilterContext* yadifFilter = nullptr;
if (m_pFrame->interlaced_frame != 0)
{
checkRc(avfilter_graph_create_filter(&yadifFilter, avfilter_get_by_name("yadif"), "thumb_deint", "deint=1", nullptr, m_pFilterGraph),
"Failed to create deinterlace filter");
}
AVFilterContext* scaleFilter = nullptr;
checkRc(avfilter_graph_create_filter(&scaleFilter, avfilter_get_by_name("scale"), "thumb_scale", createScaleString(size, maintainAspectRatio).c_str(), nullptr, m_pFilterGraph),
"Failed to create scale filter");
AVFilterContext* formatFilter = nullptr;
checkRc(avfilter_graph_create_filter(&formatFilter, avfilter_get_by_name("format"), "thumb_format", "pix_fmts=bgra", nullptr, m_pFilterGraph),
"Failed to create format filter");
AVFilterContext* rotateFilter = nullptr;
auto rotation = getStreamRotation();
if (rotation != -1)
{
checkRc(avfilter_graph_create_filter(&rotateFilter, avfilter_get_by_name("transpose"), "thumb_rotate", to_string(rotation).c_str(), nullptr, m_pFilterGraph),
"Failed to create rotate filter");
}
checkRc(avfilter_link(rotateFilter ? rotateFilter : formatFilter, 0, m_pFilterSink, 0), "Failed to link final filter");
if (rotateFilter)
{
checkRc(avfilter_link(formatFilter, 0, rotateFilter, 0), "Failed to link format filter");
}
checkRc(avfilter_link(scaleFilter, 0, formatFilter, 0), "Failed to link scale filter");
if (yadifFilter)
{
checkRc(avfilter_link(yadifFilter, 0, scaleFilter, 0), "Failed to link yadif filter");
}
checkRc(avfilter_link(m_pFilterSource, 0, yadifFilter ? yadifFilter : scaleFilter, 0), "Failed to link source filter");
checkRc(avfilter_graph_config(m_pFilterGraph, nullptr), "Failed to configure filter graph");
}
