JNIEXPORT void JNICALL Java_org_linphone_mediastream_video_capture_AndroidVideoApi5JniWrapper_putImage(JNIEnv* env, jclass thiz,jlong nativePtr,jbyteArray frame) { AndroidReaderContext* d = (AndroidReaderContext*) nativePtr; if (!d->androidCamera) return; ms_mutex_lock(&d->mutex); if (!ms_video_capture_new_frame(&d->fpsControl,d->filter->ticker->time)) { ms_mutex_unlock(&d->mutex); return; } if (d->rotation != UNDEFINED_ROTATION && d->rotationSavedDuringVSize != d->rotation) { ms_warning("Rotation has changed (new value: %d) since vsize was run (old value: %d)." "Will produce inverted images. Use set_device_orientation() then update call.\n", d->rotation, d->rotationSavedDuringVSize); } int image_rotation_correction = compute_image_rotation_correction(d, d->rotationSavedDuringVSize); jboolean isCopied; jbyte* jinternal_buff = env->GetByteArrayElements(frame, &isCopied); if (isCopied) { ms_warning("The video frame received from Java has been copied"); } int y_cropping_offset=0, cbcr_cropping_offset=0; //compute_cropping_offsets(d->hwCapableSize, d->requestedSize, &y_cropping_offset, &cbcr_cropping_offset); int width = d->hwCapableSize.width; int height = d->hwCapableSize.height; uint8_t* y_src = (uint8_t*)(jinternal_buff + y_cropping_offset); uint8_t* cbcr_src = (uint8_t*) (jinternal_buff + width * height + cbcr_cropping_offset); /* Warning note: image_rotation_correction == 90 does not imply portrait mode ! (incorrect function naming). It only implies one thing: image needs to rotated by that amount to be correctly displayed. */ mblk_t* yuv_block = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(y_src , cbcr_src , image_rotation_correction , d->requestedSize.width , d->requestedSize.height , d->hwCapableSize.width , d->hwCapableSize.width, false, d->useDownscaling); if (yuv_block) { if (d->frame) freemsg(d->frame); d->frame = yuv_block; } ms_mutex_unlock(&d->mutex); // JNI_ABORT free the buffer without copying back the possible changes env->ReleaseByteArrayElements(frame, jinternal_buff, JNI_ABORT); }
mblk_t *copy_ycbcrbiplanar_to_true_yuv_with_rotation(uint8_t* y, uint8_t * cbcr, int rotation, int w, int h, int y_byte_per_row,int cbcr_byte_per_row, bool_t uFirstvSecond) { return copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(y, cbcr, rotation, w, h, y_byte_per_row, cbcr_byte_per_row, uFirstvSecond, FALSE); }
static void test_video_processing_base (bool_t downscaling,bool_t rotate_clock_wise,bool_t flip) { MSVideoSize src_size = { MS_VIDEO_SIZE_VGA_W, MS_VIDEO_SIZE_VGA_H }; MSVideoSize dest_size = src_size; mblk_t * yuv_block2; YuvBuf yuv; int y_bytes_per_row = src_size.width + src_size.width%32 ; uint8_t* y = (uint8_t*)ms_malloc(y_bytes_per_row*src_size.height); /*to allow bloc to work with multiple of 32*/ int crcb_bytes_per_row = src_size.width/2 + (src_size.width/2)%32 ; uint8_t* cbcr = (uint8_t*)ms_malloc(crcb_bytes_per_row*src_size.height); int i,j; MSYuvBufAllocator *yba = ms_yuv_buf_allocator_new(); int factor=downscaling?2:1; int rotation = 0; if (rotate_clock_wise && flip) { ms_fatal("fix you test"); } if (rotate_clock_wise) { rotation = 90; dest_size.height=src_size.width; dest_size.width=src_size.height; } else if (flip) { rotation = 180; } dest_size.height = dest_size.height/factor; dest_size.width=dest_size.width/factor; for (i=0;i<src_size.height*src_size.width;i++) { y[i]=i%256; } for (i=0;i<src_size.height*src_size.width/2;i++) { cbcr[i]=i%256; } yuv_block2 = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(yba, y ,cbcr ,rotation , dest_size.width , dest_size.height , y_bytes_per_row , crcb_bytes_per_row , 1 , downscaling); BC_ASSERT_FALSE(ms_yuv_buf_init_from_mblk(&yuv, yuv_block2)); BC_ASSERT_EQUAL(dest_size.width,yuv.w, int, "%d"); BC_ASSERT_EQUAL(dest_size.height,yuv.h, int, "%d"); if (rotate_clock_wise) { /*check y*/ for (i=0;i<yuv.h;i++) { for (j=0;j<yuv.w;j++) if (yuv.planes[0][i*yuv.strides[0]+j] != y[(yuv.w-1-j)*factor*y_bytes_per_row+i*factor]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[(yuv.w-1-j)*factor*y_bytes_per_row+i*factor]); BC_FAIL("bad y value"); break; } } /*check cb*/ for (i=0;i<yuv.h/2;i++) { for (j=0;j<yuv.w/2;j++) { if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor]); BC_FAIL("bad cb value"); break; } if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor+1]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor+1]); BC_FAIL("bad cr value"); break; } } } } else if (flip) { /*check y*/ for (i=0;i<yuv.h;i++) { for (j=0;j<yuv.w;j++) if (yuv.planes[0][i*yuv.strides[0]+j] != y[(yuv.h-1-i)*factor*y_bytes_per_row+(yuv.w-1-j)*factor]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[(yuv.h-1-i)*factor*y_bytes_per_row+(yuv.w-1-j)*factor]); BC_FAIL("bad y value"); break; } } for (i=0;i<yuv.h/2;i++) { for (j=0;j<yuv.w/2;j++) { /*check cb*/ if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor]); BC_FAIL("bad cb value"); break; } /*check cr*/ if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor+1]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor+1]); BC_FAIL("bad cr value"); break; } } } } else { /*check y*/ for (i=0;i<yuv.h;i++) { for (j=0;j<yuv.w;j++) if (yuv.planes[0][i*yuv.strides[0]+j] != y[i*factor*y_bytes_per_row+j*factor]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[i*factor*y_bytes_per_row+j*factor]); BC_FAIL("bad y value"); break; } } for (i=0;i<yuv.h/2;i++) { for (j=0;j<yuv.w/2;j++) { /*check cb*/ if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[i*factor*crcb_bytes_per_row+2*j*factor]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,cbcr[i*factor*crcb_bytes_per_row+2*j*factor]); BC_FAIL("bad cb value"); break; } /*check cr*/ if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[i*factor*crcb_bytes_per_row+2*j*factor+1]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,cbcr[i*factor*crcb_bytes_per_row+2*j*factor+1]); BC_FAIL("bad cr value"); break; } } } } freemsg(yuv_block2); ms_free(y); ms_free(cbcr); ms_yuv_buf_allocator_free(yba); }
static void dec_process(MSFilter *f){ DecData *d=(DecData*)f->data; MSPicture pic = {0}; mblk_t *im,*om = NULL; ssize_t oBufidx = -1; size_t bufsize; bool_t need_reinit=FALSE; bool_t request_pli=FALSE; MSQueue nalus; AMediaCodecBufferInfo info; ms_queue_init(&nalus); while((im=ms_queue_get(f->inputs[0]))!=NULL){ if (d->packet_num==0 && d->sps && d->pps){ mblk_set_timestamp_info(d->sps,mblk_get_timestamp_info(im)); mblk_set_timestamp_info(d->pps,mblk_get_timestamp_info(im)); rfc3984_unpack(&d->unpacker, d->sps, &nalus); rfc3984_unpack(&d->unpacker, d->pps, &nalus); d->sps=NULL; d->pps=NULL; } if(rfc3984_unpack(&d->unpacker,im,&nalus) <0){ request_pli=TRUE; } if (!ms_queue_empty(&nalus)){ int size; uint8_t *buf=NULL; ssize_t iBufidx; size=nalusToFrame(d,&nalus,&need_reinit); if (need_reinit) { //In case of rotation, the decoder needs to flushed in order to restart with the new video size AMediaCodec_flush(d->codec); d->first_buffer_queued = FALSE; } /*First put our H264 bitstream into the decoder*/ iBufidx = AMediaCodec_dequeueInputBuffer(d->codec, TIMEOUT_US); if (iBufidx >= 0) { buf = AMediaCodec_getInputBuffer(d->codec, iBufidx, &bufsize); if(buf == NULL) { ms_error("MSMediaCodecH264Dec: AMediaCodec_getInputBuffer() returned NULL"); break; } if((size_t)size > bufsize) { ms_error("Cannot copy the bitstream into the input buffer size : %i and bufsize %i",size,(int) bufsize); break; } else { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); memcpy(buf,d->bitstream,(size_t)size); AMediaCodec_queueInputBuffer(d->codec, iBufidx, 0, (size_t)size, (ts.tv_nsec/1000) + 10000LL, 0); d->first_buffer_queued = TRUE; } }else if (iBufidx == AMEDIA_ERROR_UNKNOWN){ ms_error("MSMediaCodecH264Dec: AMediaCodec_dequeueInputBuffer() had an exception"); } } d->packet_num++; if (d->sps && d->pps) request_pli = FALSE; else request_pli = TRUE; } /*secondly try to get decoded frames from the decoder, this is performed every tick*/ while (d->first_buffer_queued && (oBufidx = AMediaCodec_dequeueOutputBuffer(d->codec, &info, TIMEOUT_US)) >= 0){ AMediaFormat *format; int width = 0, height = 0, color = 0; uint8_t *buf = AMediaCodec_getOutputBuffer(d->codec, oBufidx, &bufsize); if(buf == NULL){ ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_DECODING_ERRORS); ms_error("MSMediaCodecH264Dec: AMediaCodec_getOutputBuffer() returned NULL"); } format = AMediaCodec_getOutputFormat(d->codec); if(format != NULL){ AMediaFormat_getInt32(format, "width", &width); AMediaFormat_getInt32(format, "height", &height); AMediaFormat_getInt32(format, "color-format", &color); d->vsize.width=width; d->vsize.height=height; AMediaFormat_delete(format); } if(buf != NULL && d->sps && d->pps){ /*some decoders output garbage while no sps or pps have been received yet !*/ if(width != 0 && height != 0 ){ if(color == 19) { //YUV int ysize = width*height; int usize = ysize/4; om = ms_yuv_buf_allocator_get(d->buf_allocator,&pic,width,height); memcpy(pic.planes[0],buf,ysize); memcpy(pic.planes[1],buf+ysize,usize); memcpy(pic.planes[2],buf+ysize+usize,usize); } else { uint8_t* cbcr_src = (uint8_t*) (buf + width * height); om = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(d->buf_allocator, buf, cbcr_src, 0, width, height, width, width, TRUE, FALSE); } if (!d->first_image_decoded) { ms_message("First frame decoded %ix%i",width,height); d->first_image_decoded = true; ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED); } ms_queue_put(f->outputs[0], om); }else{ ms_error("MSMediaCodecH264Dec: width and height are not known !"); } } AMediaCodec_releaseOutputBuffer(d->codec, oBufidx, FALSE); } if (oBufidx == AMEDIA_ERROR_UNKNOWN){ ms_error("MSMediaCodecH264Dec: AMediaCodec_dequeueOutputBuffer() had an exception"); } if (d->avpf_enabled && request_pli) { ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI); } ms_queue_flush(f->inputs[0]); }
static void test_video_processing (void) { MSVideoSize src_size = { MS_VIDEO_SIZE_VGA_W, MS_VIDEO_SIZE_VGA_H }; MSVideoSize src_dest = { MS_VIDEO_SIZE_VGA_W, MS_VIDEO_SIZE_VGA_H }; mblk_t * yuv_block2; YuvBuf yuv; int y_bytes_per_row = src_size.width + src_size.width%32 ; uint8_t* y = (uint8_t*)ms_malloc(y_bytes_per_row*src_size.height); /*to allow bloc to work with multiple of 32*/ int crcb_bytes_per_row = src_size.width/2 + (src_size.width/2)%32 ; uint8_t* cbcr = (uint8_t*)ms_malloc(crcb_bytes_per_row*src_size.height); int i,j; for (i=0;i<src_size.height*src_size.width;i++) { y[i]=i%256; } for (i=0;i<src_size.height*src_size.width/2;i++) { cbcr[i]=i%256; } yuv_block2 = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2( y ,cbcr ,0 , src_size.width , src_size.height , y_bytes_per_row , crcb_bytes_per_row , 1 , 0); CU_ASSERT_FALSE(ms_yuv_buf_init_from_mblk(&yuv, yuv_block2)); CU_ASSERT_EQUAL(src_dest.width,yuv.w); CU_ASSERT_EQUAL(src_dest.height,yuv.h); /*check y*/ for (i=0;i<yuv.h;i++) { for (j=0;j<yuv.w;j++) if (yuv.planes[0][i*yuv.strides[0]+j] != y[i*y_bytes_per_row+j]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[i*y_bytes_per_row+j]); CU_FAIL("bad y value"); break; } } /*check cb*/ for (i=0;i<yuv.h/2;i++) { for (j=0;j<yuv.w/2;j++) if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[i*crcb_bytes_per_row+2*j]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,y[i*crcb_bytes_per_row+2*j]); CU_FAIL("bad cb value"); break; } } /*check cr*/ for (i=0;i<yuv.h/2;i++) { for (j=0;j<yuv.w/2;j++) if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[i*crcb_bytes_per_row+2*j+1]) { ms_error("Wrong value [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,y[i*crcb_bytes_per_row+2*j+1]); CU_FAIL("bad cr value"); break; } } ms_free(y); ms_free(cbcr); }
static void dec_process(MSFilter *f){ DecData *d=(DecData*)f->data; MSPicture pic = {0}; mblk_t *im,*om = NULL; bool_t need_reinit=FALSE; bool_t request_pli=FALSE; MSQueue nalus; ms_queue_init(&nalus); while((im=ms_queue_get(f->inputs[0]))!=NULL){ if (d->packet_num==0 && d->sps && d->pps){ mblk_set_timestamp_info(d->sps,mblk_get_timestamp_info(im)); mblk_set_timestamp_info(d->pps,mblk_get_timestamp_info(im)); rfc3984_unpack(&d->unpacker, d->sps, &nalus); rfc3984_unpack(&d->unpacker, d->pps, &nalus); d->sps=NULL; d->pps=NULL; } if(rfc3984_unpack(&d->unpacker,im,&nalus) <0){ request_pli=TRUE; } if (!ms_queue_empty(&nalus)){ AMediaCodecBufferInfo info; int size; int width = 0, height = 0, color = 0; uint8_t *buf=NULL; size_t bufsize; ssize_t iBufidx, oBufidx; size=nalusToFrame(d,&nalus,&need_reinit); if (need_reinit) { //In case of rotation, the decoder needs to flushed in order to restart with the new video size AMediaCodec_flush(d->codec); } iBufidx = AMediaCodec_dequeueInputBuffer(d->codec, TIMEOUT_US); if (iBufidx >= 0) { buf = AMediaCodec_getInputBuffer(d->codec, iBufidx, &bufsize); if(buf == NULL) { break; } if((size_t)size > bufsize) { ms_error("Cannot copy the bitstream into the input buffer size : %i and bufsize %i",size,(int) bufsize); } else { memcpy(buf,d->bitstream,(size_t)size); AMediaCodec_queueInputBuffer(d->codec, iBufidx, 0, (size_t)size, TIMEOUT_US, 0); } } oBufidx = AMediaCodec_dequeueOutputBuffer(d->codec, &info, TIMEOUT_US); if(oBufidx >= 0){ AMediaFormat *format; buf = AMediaCodec_getOutputBuffer(d->codec, oBufidx, &bufsize); if(buf == NULL){ ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_DECODING_ERRORS); break; } format = AMediaCodec_getOutputFormat(d->codec); if(format != NULL){ AMediaFormat_getInt32(format, "width", &width); AMediaFormat_getInt32(format, "height", &height); AMediaFormat_getInt32(format, "color-format", &color); d->vsize.width=width; d->vsize.height=height; AMediaFormat_delete(format); } } if(buf != NULL){ //YUV if(width != 0 && height != 0 ){ if(color == 19) { int ysize = width*height; int usize = ysize/4; om=ms_yuv_buf_alloc(&pic,width,height); memcpy(pic.planes[0],buf,ysize); memcpy(pic.planes[1],buf+ysize,usize); memcpy(pic.planes[2],buf+ysize+usize,usize); } else { uint8_t* cbcr_src = (uint8_t*) (buf + width * height); om = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(d->buf_allocator, buf, cbcr_src, 0, width, height, width, width, TRUE, FALSE); } if (!d->first_image_decoded) { ms_message("First frame decoded %ix%i",width,height); d->first_image_decoded = true; ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED); } ms_queue_put(f->outputs[0], om); } if(oBufidx > 0) { AMediaCodec_releaseOutputBuffer(d->codec, oBufidx, FALSE); } } } d->packet_num++; } if (d->avpf_enabled && request_pli) { ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI); } }