Exemplo n.º 1
0
JNIEXPORT void JNICALL Java_org_linphone_mediastream_video_capture_AndroidVideoApi5JniWrapper_putImage(JNIEnv*  env,
		jclass  thiz,jlong nativePtr,jbyteArray frame) {
	AndroidReaderContext* d = (AndroidReaderContext*) nativePtr;
	if (!d->androidCamera)
		return;
	ms_mutex_lock(&d->mutex);

	if (!ms_video_capture_new_frame(&d->fpsControl,d->filter->ticker->time)) {
		ms_mutex_unlock(&d->mutex);
		return;
	}

	if (d->rotation != UNDEFINED_ROTATION && d->rotationSavedDuringVSize != d->rotation) {
		ms_warning("Rotation has changed (new value: %d) since vsize was run (old value: %d)."
					"Will produce inverted images. Use set_device_orientation() then update call.\n",
			d->rotation, d->rotationSavedDuringVSize);
	}

	int image_rotation_correction = compute_image_rotation_correction(d, d->rotationSavedDuringVSize);

	jboolean isCopied;
	jbyte* jinternal_buff = env->GetByteArrayElements(frame, &isCopied);
	if (isCopied) {
		ms_warning("The video frame received from Java has been copied");
	}

	int y_cropping_offset=0, cbcr_cropping_offset=0;
	//compute_cropping_offsets(d->hwCapableSize, d->requestedSize, &y_cropping_offset, &cbcr_cropping_offset);

	int width = d->hwCapableSize.width;
	int height = d->hwCapableSize.height;

	uint8_t* y_src = (uint8_t*)(jinternal_buff + y_cropping_offset);
	uint8_t* cbcr_src = (uint8_t*) (jinternal_buff + width * height + cbcr_cropping_offset);

	/* Warning note: image_rotation_correction == 90 does not imply portrait mode !
	   (incorrect function naming).
	   It only implies one thing: image needs to rotated by that amount to be correctly
	   displayed.
	*/
 	mblk_t* yuv_block = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(y_src
														, cbcr_src
														, image_rotation_correction
														, d->requestedSize.width
														, d->requestedSize.height
														, d->hwCapableSize.width
														, d->hwCapableSize.width,
														false,
														d->useDownscaling);
	if (yuv_block) {
		if (d->frame)
			freemsg(d->frame);
		d->frame = yuv_block;
	}
	ms_mutex_unlock(&d->mutex);

	// JNI_ABORT free the buffer without copying back the possible changes
	env->ReleaseByteArrayElements(frame, jinternal_buff, JNI_ABORT);
}
Exemplo n.º 2
0
mblk_t *copy_ycbcrbiplanar_to_true_yuv_with_rotation(uint8_t* y, uint8_t * cbcr, int rotation, int w, int h, int y_byte_per_row,int cbcr_byte_per_row, bool_t uFirstvSecond) {
	return copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(y, cbcr, rotation, w, h, y_byte_per_row, cbcr_byte_per_row, uFirstvSecond, FALSE);
}
static void test_video_processing_base (bool_t downscaling,bool_t rotate_clock_wise,bool_t flip) {
	MSVideoSize src_size = { MS_VIDEO_SIZE_VGA_W, MS_VIDEO_SIZE_VGA_H };
	MSVideoSize dest_size = src_size;
	
	mblk_t * yuv_block2;
	YuvBuf yuv;
	int y_bytes_per_row = src_size.width + src_size.width%32 ;
	uint8_t* y = (uint8_t*)ms_malloc(y_bytes_per_row*src_size.height); /*to allow bloc to work with multiple of 32*/
	int crcb_bytes_per_row = src_size.width/2 + (src_size.width/2)%32 ;
	uint8_t* cbcr = (uint8_t*)ms_malloc(crcb_bytes_per_row*src_size.height);
	int i,j;
	MSYuvBufAllocator *yba = ms_yuv_buf_allocator_new();
	int factor=downscaling?2:1;
	int rotation = 0;
	if (rotate_clock_wise && flip) {
		ms_fatal("fix you test");
	}
	if (rotate_clock_wise) {
		rotation = 90;
		dest_size.height=src_size.width;
		dest_size.width=src_size.height;
	} else if (flip) {
		rotation = 180;
	}
	dest_size.height = dest_size.height/factor;
	dest_size.width=dest_size.width/factor;
	
	for (i=0;i<src_size.height*src_size.width;i++) {
		y[i]=i%256;
	}
	for (i=0;i<src_size.height*src_size.width/2;i++) {
		cbcr[i]=i%256;
	}

	yuv_block2 = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(yba,	y
																					,cbcr
																					,rotation
																					, dest_size.width
																					, dest_size.height
																					, y_bytes_per_row
																					, crcb_bytes_per_row
																					, 1
																					, downscaling);

	BC_ASSERT_FALSE(ms_yuv_buf_init_from_mblk(&yuv, yuv_block2));
	BC_ASSERT_EQUAL(dest_size.width,yuv.w, int, "%d");
	BC_ASSERT_EQUAL(dest_size.height,yuv.h, int, "%d");

	if (rotate_clock_wise) {
		/*check y*/
		for (i=0;i<yuv.h;i++) {
			for (j=0;j<yuv.w;j++)
				if (yuv.planes[0][i*yuv.strides[0]+j] != y[(yuv.w-1-j)*factor*y_bytes_per_row+i*factor]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[(yuv.w-1-j)*factor*y_bytes_per_row+i*factor]);
					BC_FAIL("bad y value");
					break;
				}
			
		}
		/*check cb*/
		for (i=0;i<yuv.h/2;i++) {
			for (j=0;j<yuv.w/2;j++) {
				if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor]);
					BC_FAIL("bad cb value");
					break;
				}
				if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor+1]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,cbcr[(yuv.w/2-1-j)*factor*crcb_bytes_per_row+2*i*factor+1]);
					BC_FAIL("bad cr value");
					break;
				}
			}
		}
	} else if (flip) {
		
		/*check y*/
		for (i=0;i<yuv.h;i++) {
			for (j=0;j<yuv.w;j++)
				if (yuv.planes[0][i*yuv.strides[0]+j] != y[(yuv.h-1-i)*factor*y_bytes_per_row+(yuv.w-1-j)*factor]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[(yuv.h-1-i)*factor*y_bytes_per_row+(yuv.w-1-j)*factor]);
					BC_FAIL("bad y value");
					break;
				}
		}
		
		for (i=0;i<yuv.h/2;i++) {
			for (j=0;j<yuv.w/2;j++) {
				/*check cb*/
				if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor]);
					BC_FAIL("bad cb value");
					break;
				}
				/*check cr*/
				if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor+1]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,cbcr[(yuv.h/2-1-i)*factor*crcb_bytes_per_row+2*(yuv.w/2-1-j)*factor+1]);
					BC_FAIL("bad cr value");
					break;
				}
				
			}
		}
	}
	else {
		/*check y*/
		for (i=0;i<yuv.h;i++) {
			for (j=0;j<yuv.w;j++)
				if (yuv.planes[0][i*yuv.strides[0]+j] != y[i*factor*y_bytes_per_row+j*factor]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[i*factor*y_bytes_per_row+j*factor]);
					BC_FAIL("bad y value");
					break;
				}
		}
		
		for (i=0;i<yuv.h/2;i++) {
			for (j=0;j<yuv.w/2;j++) {
				/*check cb*/
				if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[i*factor*crcb_bytes_per_row+2*j*factor]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,cbcr[i*factor*crcb_bytes_per_row+2*j*factor]);
					BC_FAIL("bad cb value");
					break;
				}
				/*check cr*/
				if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[i*factor*crcb_bytes_per_row+2*j*factor+1]) {
					ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,cbcr[i*factor*crcb_bytes_per_row+2*j*factor+1]);
					BC_FAIL("bad cr value");
					break;
				}

			}
		}
	}
	freemsg(yuv_block2);
	ms_free(y);
	ms_free(cbcr);
	ms_yuv_buf_allocator_free(yba);

}
static void dec_process(MSFilter *f){
	DecData *d=(DecData*)f->data;
	MSPicture pic = {0};
	mblk_t *im,*om = NULL;
	ssize_t oBufidx = -1;
	size_t bufsize;
	bool_t need_reinit=FALSE;
	bool_t request_pli=FALSE;
	MSQueue nalus;
	AMediaCodecBufferInfo info;
	
	ms_queue_init(&nalus);

	while((im=ms_queue_get(f->inputs[0]))!=NULL){
		if (d->packet_num==0 && d->sps && d->pps){
			mblk_set_timestamp_info(d->sps,mblk_get_timestamp_info(im));
			mblk_set_timestamp_info(d->pps,mblk_get_timestamp_info(im));
			rfc3984_unpack(&d->unpacker, d->sps, &nalus);
            rfc3984_unpack(&d->unpacker, d->pps, &nalus);
			d->sps=NULL;
			d->pps=NULL;
		}

		if(rfc3984_unpack(&d->unpacker,im,&nalus) <0){
			request_pli=TRUE;
		}
		if (!ms_queue_empty(&nalus)){
			int size;
			uint8_t *buf=NULL;
			ssize_t iBufidx;

			size=nalusToFrame(d,&nalus,&need_reinit);

			if (need_reinit) {
				//In case of rotation, the decoder needs to flushed in order to restart with the new video size
				AMediaCodec_flush(d->codec);
				d->first_buffer_queued = FALSE;
			}

			/*First put our H264 bitstream into the decoder*/
			iBufidx = AMediaCodec_dequeueInputBuffer(d->codec, TIMEOUT_US);
			if (iBufidx >= 0) {
				buf = AMediaCodec_getInputBuffer(d->codec, iBufidx, &bufsize);
				if(buf == NULL) {
					ms_error("MSMediaCodecH264Dec: AMediaCodec_getInputBuffer() returned NULL");
					break;
				}
				if((size_t)size > bufsize) {
					ms_error("Cannot copy the bitstream into the input buffer size : %i and bufsize %i",size,(int) bufsize);
					break;
				} else {
					struct timespec ts;
					clock_gettime(CLOCK_MONOTONIC, &ts);
					memcpy(buf,d->bitstream,(size_t)size);
					AMediaCodec_queueInputBuffer(d->codec, iBufidx, 0, (size_t)size, (ts.tv_nsec/1000) + 10000LL, 0);
					d->first_buffer_queued = TRUE;
				}
			}else if (iBufidx == AMEDIA_ERROR_UNKNOWN){
				ms_error("MSMediaCodecH264Dec: AMediaCodec_dequeueInputBuffer() had an exception");
			}
		}
		d->packet_num++;
		if (d->sps && d->pps) request_pli = FALSE;
		else request_pli = TRUE;
	}
	
	/*secondly try to get decoded frames from the decoder, this is performed every tick*/
	while (d->first_buffer_queued && (oBufidx = AMediaCodec_dequeueOutputBuffer(d->codec, &info, TIMEOUT_US)) >= 0){
		AMediaFormat *format;
		int width = 0, height = 0, color = 0;
		uint8_t *buf = AMediaCodec_getOutputBuffer(d->codec, oBufidx, &bufsize);
		
		if(buf == NULL){
			ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_DECODING_ERRORS);
			ms_error("MSMediaCodecH264Dec: AMediaCodec_getOutputBuffer() returned NULL");
		}

		format = AMediaCodec_getOutputFormat(d->codec);
		if(format != NULL){
			AMediaFormat_getInt32(format, "width", &width);
			AMediaFormat_getInt32(format, "height", &height);
			AMediaFormat_getInt32(format, "color-format", &color);

			d->vsize.width=width;
			d->vsize.height=height;
			AMediaFormat_delete(format);
		}

		if(buf != NULL && d->sps && d->pps){ /*some decoders output garbage while no sps or pps have been received yet !*/
			if(width != 0 && height != 0 ){
				if(color == 19) {
					//YUV
					int ysize = width*height;
					int usize = ysize/4;
					om = ms_yuv_buf_allocator_get(d->buf_allocator,&pic,width,height);
					memcpy(pic.planes[0],buf,ysize);
					memcpy(pic.planes[1],buf+ysize,usize);
					memcpy(pic.planes[2],buf+ysize+usize,usize);
				} else {
					uint8_t* cbcr_src = (uint8_t*) (buf + width * height);
					om = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(d->buf_allocator, buf, cbcr_src, 0, width, height, width, width, TRUE, FALSE);
				}

				if (!d->first_image_decoded) {
					ms_message("First frame decoded %ix%i",width,height);
					d->first_image_decoded = true;
					ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
				}
				ms_queue_put(f->outputs[0], om);
			}else{
				ms_error("MSMediaCodecH264Dec: width and height are not known !");
			}
		}
		AMediaCodec_releaseOutputBuffer(d->codec, oBufidx, FALSE);
	}
	if (oBufidx == AMEDIA_ERROR_UNKNOWN){
		ms_error("MSMediaCodecH264Dec: AMediaCodec_dequeueOutputBuffer() had an exception");
	}

	if (d->avpf_enabled && request_pli) {
    	ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI);
    }
    ms_queue_flush(f->inputs[0]);
}
static void test_video_processing (void) {
	MSVideoSize src_size = { MS_VIDEO_SIZE_VGA_W, MS_VIDEO_SIZE_VGA_H };
	MSVideoSize src_dest = { MS_VIDEO_SIZE_VGA_W, MS_VIDEO_SIZE_VGA_H };
	mblk_t * yuv_block2;
	YuvBuf yuv;
	int y_bytes_per_row = src_size.width + src_size.width%32 ;
	uint8_t* y = (uint8_t*)ms_malloc(y_bytes_per_row*src_size.height); /*to allow bloc to work with multiple of 32*/
	int crcb_bytes_per_row = src_size.width/2 + (src_size.width/2)%32 ;
	uint8_t* cbcr = (uint8_t*)ms_malloc(crcb_bytes_per_row*src_size.height);
	int i,j;

	for (i=0;i<src_size.height*src_size.width;i++) {
		y[i]=i%256;
	}
	for (i=0;i<src_size.height*src_size.width/2;i++) {
		cbcr[i]=i%256;
	}

	yuv_block2 = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(	y
																					,cbcr
																					,0
																					, src_size.width
																					, src_size.height
																					, y_bytes_per_row
																					, crcb_bytes_per_row
																					, 1
																					, 0);

	CU_ASSERT_FALSE(ms_yuv_buf_init_from_mblk(&yuv, yuv_block2));

	CU_ASSERT_EQUAL(src_dest.width,yuv.w);
	CU_ASSERT_EQUAL(src_dest.height,yuv.h);

	/*check y*/
	for (i=0;i<yuv.h;i++) {
		for (j=0;j<yuv.w;j++)
		if (yuv.planes[0][i*yuv.strides[0]+j] != y[i*y_bytes_per_row+j]) {
			ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[0][i*yuv.strides[0]+j],i*yuv.strides[0]+j,y[i*y_bytes_per_row+j]);
			CU_FAIL("bad y value");
			break;
		}
	}

	/*check cb*/
	for (i=0;i<yuv.h/2;i++) {
		for (j=0;j<yuv.w/2;j++)
		if (yuv.planes[1][i*yuv.strides[1]+j] != cbcr[i*crcb_bytes_per_row+2*j]) {
			ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[1][i*yuv.strides[1]+j],i*yuv.strides[1]+j,y[i*crcb_bytes_per_row+2*j]);
			CU_FAIL("bad cb value");
			break;
		}
	}

	/*check cr*/
	for (i=0;i<yuv.h/2;i++) {
		for (j=0;j<yuv.w/2;j++)
		if (yuv.planes[2][i*yuv.strides[2]+j] != cbcr[i*crcb_bytes_per_row+2*j+1]) {
			ms_error("Wrong value  [%i] at ofset [%i], should be [%i]",yuv.planes[2][i*yuv.strides[2]+j],i*yuv.strides[2]+j,y[i*crcb_bytes_per_row+2*j+1]);
			CU_FAIL("bad cr value");
			break;
		}
	}

	ms_free(y);
	ms_free(cbcr);

}
Exemplo n.º 6
0
static void dec_process(MSFilter *f){
	DecData *d=(DecData*)f->data;
	MSPicture pic = {0};
	mblk_t *im,*om = NULL;
	bool_t need_reinit=FALSE;
	bool_t request_pli=FALSE;
	MSQueue nalus;
	ms_queue_init(&nalus);

	while((im=ms_queue_get(f->inputs[0]))!=NULL){
		if (d->packet_num==0 && d->sps && d->pps){
			mblk_set_timestamp_info(d->sps,mblk_get_timestamp_info(im));
			mblk_set_timestamp_info(d->pps,mblk_get_timestamp_info(im));
			rfc3984_unpack(&d->unpacker, d->sps, &nalus);
            rfc3984_unpack(&d->unpacker, d->pps, &nalus);
			d->sps=NULL;
			d->pps=NULL;
		}

		if(rfc3984_unpack(&d->unpacker,im,&nalus) <0){
			request_pli=TRUE;
		}
		if (!ms_queue_empty(&nalus)){
			AMediaCodecBufferInfo info;
			int size;
			int width = 0, height = 0, color = 0;
			uint8_t *buf=NULL;
			size_t bufsize;
			ssize_t iBufidx, oBufidx;

			size=nalusToFrame(d,&nalus,&need_reinit);

			if (need_reinit) {
				//In case of rotation, the decoder needs to flushed in order to restart with the new video size
				AMediaCodec_flush(d->codec);
			}

			iBufidx = AMediaCodec_dequeueInputBuffer(d->codec, TIMEOUT_US);
			if (iBufidx >= 0) {
				buf = AMediaCodec_getInputBuffer(d->codec, iBufidx, &bufsize);
				if(buf == NULL) {
					break;
				}
				if((size_t)size > bufsize) {
					ms_error("Cannot copy the bitstream into the input buffer size : %i and bufsize %i",size,(int) bufsize);
				} else {
					memcpy(buf,d->bitstream,(size_t)size);
					AMediaCodec_queueInputBuffer(d->codec, iBufidx, 0, (size_t)size, TIMEOUT_US, 0);
				}
			}

            oBufidx = AMediaCodec_dequeueOutputBuffer(d->codec, &info, TIMEOUT_US);
			if(oBufidx >= 0){
				AMediaFormat *format;
				buf = AMediaCodec_getOutputBuffer(d->codec, oBufidx, &bufsize);
				if(buf == NULL){
					ms_filter_notify_no_arg(f,MS_VIDEO_DECODER_DECODING_ERRORS);
					break;
				}

				format = AMediaCodec_getOutputFormat(d->codec);
				if(format != NULL){
					AMediaFormat_getInt32(format, "width", &width);
					AMediaFormat_getInt32(format, "height", &height);
					AMediaFormat_getInt32(format, "color-format", &color);

					d->vsize.width=width;
					d->vsize.height=height;
					AMediaFormat_delete(format);
				}
			}

			if(buf != NULL){
				//YUV
				if(width != 0 && height != 0 ){
					if(color == 19) {
						int ysize = width*height;
						int usize = ysize/4;
						om=ms_yuv_buf_alloc(&pic,width,height);
						memcpy(pic.planes[0],buf,ysize);
						memcpy(pic.planes[1],buf+ysize,usize);
						memcpy(pic.planes[2],buf+ysize+usize,usize);
					} else {
						uint8_t* cbcr_src = (uint8_t*) (buf + width * height);
						om = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(d->buf_allocator, buf, cbcr_src, 0, width, height, width, width, TRUE, FALSE);
					}

					if (!d->first_image_decoded) {
						ms_message("First frame decoded %ix%i",width,height);
						d->first_image_decoded = true;
						ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
					}

					ms_queue_put(f->outputs[0], om);
				}

				if(oBufidx > 0) {
                	AMediaCodec_releaseOutputBuffer(d->codec, oBufidx, FALSE);
				}


			}
		}
		d->packet_num++;
	}

	if (d->avpf_enabled && request_pli) {
    	ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI);
    }
}