Gesture* DataProcesser::analyze(DataAnalyzer<Angles> data, int new_samples) {
Gesture* gb;
if (find_beat(data, &gb))
return gb;
else if (find_motion(data, &gb))
return gb;
else
return nullptr;
}
static int filter_frame(AVFilterLink *link, AVFrame *in)
{
DeshakeContext *deshake = link->dst->priv;
AVFilterLink *outlink = link->dst->outputs[0];
AVFrame *out;
Transform t = {{0},0}, orig = {{0},0};
float matrix_y[9], matrix_uv[9];
float alpha = 2.0 / deshake->refcount;
char tmp[256];
int ret = 0;
out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
if (!out) {
av_frame_free(&in);
return AVERROR(ENOMEM);
}
av_frame_copy_props(out, in);
if (CONFIG_OPENCL && deshake->opencl) {
ret = ff_opencl_deshake_process_inout_buf(link->dst,in, out);
if (ret < 0)
return ret;
}
if (deshake->cx < 0 || deshake->cy < 0 || deshake->cw < 0 || deshake->ch < 0) {
// Find the most likely global motion for the current frame
find_motion(deshake, (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0], in->data[0], link->w, link->h, in->linesize[0], &t);
} else {
uint8_t *src1 = (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0];
uint8_t *src2 = in->data[0];
deshake->cx = FFMIN(deshake->cx, link->w);
deshake->cy = FFMIN(deshake->cy, link->h);
if ((unsigned)deshake->cx + (unsigned)deshake->cw > link->w) deshake->cw = link->w - deshake->cx;
if ((unsigned)deshake->cy + (unsigned)deshake->ch > link->h) deshake->ch = link->h - deshake->cy;
// Quadword align right margin
deshake->cw &= ~15;
src1 += deshake->cy * in->linesize[0] + deshake->cx;
src2 += deshake->cy * in->linesize[0] + deshake->cx;
find_motion(deshake, src1, src2, deshake->cw, deshake->ch, in->linesize[0], &t);
}
// Copy transform so we can output it later to compare to the smoothed value
orig.vector.x = t.vector.x;
orig.vector.y = t.vector.y;
orig.angle = t.angle;
orig.zoom = t.zoom;
// Generate a one-sided moving exponential average
deshake->avg.vector.x = alpha * t.vector.x + (1.0 - alpha) * deshake->avg.vector.x;
deshake->avg.vector.y = alpha * t.vector.y + (1.0 - alpha) * deshake->avg.vector.y;
deshake->avg.angle = alpha * t.angle + (1.0 - alpha) * deshake->avg.angle;
deshake->avg.zoom = alpha * t.zoom + (1.0 - alpha) * deshake->avg.zoom;
// Remove the average from the current motion to detect the motion that
// is not on purpose, just as jitter from bumping the camera
t.vector.x -= deshake->avg.vector.x;
t.vector.y -= deshake->avg.vector.y;
t.angle -= deshake->avg.angle;
t.zoom -= deshake->avg.zoom;
// Invert the motion to undo it
t.vector.x *= -1;
t.vector.y *= -1;
t.angle *= -1;
// Write statistics to file
if (deshake->fp) {
snprintf(tmp, 256, "%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f\n", orig.vector.x, deshake->avg.vector.x, t.vector.x, orig.vector.y, deshake->avg.vector.y, t.vector.y, orig.angle, deshake->avg.angle, t.angle, orig.zoom, deshake->avg.zoom, t.zoom);
fwrite(tmp, sizeof(char), strlen(tmp), deshake->fp);
}
// Turn relative current frame motion into absolute by adding it to the
// last absolute motion
t.vector.x += deshake->last.vector.x;
t.vector.y += deshake->last.vector.y;
t.angle += deshake->last.angle;
t.zoom += deshake->last.zoom;
// Shrink motion by 10% to keep things centered in the camera frame
t.vector.x *= 0.9;
t.vector.y *= 0.9;
t.angle *= 0.9;
// Store the last absolute motion information
deshake->last.vector.x = t.vector.x;
deshake->last.vector.y = t.vector.y;
deshake->last.angle = t.angle;
deshake->last.zoom = t.zoom;
// Generate a luma transformation matrix
avfilter_get_matrix(t.vector.x, t.vector.y, t.angle, 1.0 + t.zoom / 100.0, matrix_y);
// Generate a chroma transformation matrix
avfilter_get_matrix(t.vector.x / (link->w / CHROMA_WIDTH(link)), t.vector.y / (link->h / CHROMA_HEIGHT(link)), t.angle, 1.0 + t.zoom / 100.0, matrix_uv);
// Transform the luma and chroma planes
ret = deshake->transform(link->dst, link->w, link->h, CHROMA_WIDTH(link), CHROMA_HEIGHT(link),
matrix_y, matrix_uv, INTERPOLATE_BILINEAR, deshake->edge, in, out);
// Cleanup the old reference frame
av_frame_free(&deshake->ref);
if (ret < 0)
return ret;
// Store the current frame as the reference frame for calculating the
// motion of the next frame
deshake->ref = in;
return ff_filter_frame(outlink, out);
}
int
main(int argc, char *argv[])
{
if (argc != 4)
{
fprintf(stderr,
"Usage: %s <bmp frame 1> <bmp frame 2> <bmp MC out>\n",
argv[0]);
return -1;
}
clock_t start_time = clock();
int err_code = 0;
try {
// Read the input image
bmp_in in[2];
if ((err_code = bmp_in__open(&in[0], argv[1])) != 0)
throw err_code;
if ((err_code = bmp_in__open(&in[1], argv[2])) != 0)
throw err_code;
int width = in[0].cols, height = in[0].rows;
if ((width != in[1].cols) || (height != in[1].rows))
{
fprintf(stderr, "The two input frames have different dimensions.\n");
return -1;
}
my_image_comp mono[2];
mono[0].init(height, width, 4); // Leave a border of 4 (in case needed)
mono[1].init(height, width, 4); // Leave a border of 4 (in case needed)
mono[0].perform_boundary_extension();
int n, r, c;
int num_comps = in[0].num_components;
io_byte *line = new io_byte[width*num_comps];
for (n = 0; n < 2; n++)
{
for (r = height - 1; r >= 0; r--)
{ // "r" holds the true row index we are reading, since the image
// is stored upside down in the BMP file.
if ((err_code = bmp_in__get_line(&(in[n]), line)) != 0)
throw err_code;
io_byte *src = line; // Points to first sample of component n
int *dst = mono[n].buf + r * mono[n].stride;
for (c = 0; c < width; c++, src += num_comps)
dst[c] = *src;
}
bmp_in__close(&(in[n]));
}
// Allocate storage for the motion compensated output
my_image_comp output;
output.init(height, width, 0); // Don't need a border for output
my_image_comp inter,ref_frame;
inter.init(height * 2, width * 2, 4);
ref_frame.init(height * 4, width * 4, 4);
interpolation(&mono[0], &inter);
interpolation(&inter, &ref_frame);
my_image_comp output_comps[3];
for (int i = 0; i < 3; ++i) {
output_comps[i].init(height, width, 0);
}
for (int r = 0; r < height; ++r)
for (int c = 0; c < width; ++c) {
output_comps[0].buf[r * output_comps[0].stride + c] = (mono[1].buf[r * mono[1].stride + c] >> 1) + 128;
output_comps[1].buf[r * output_comps[1].stride + c] = (mono[1].buf[r * mono[1].stride + c] >> 1) + 128;
output_comps[2].buf[r * output_comps[2].stride + c] = (mono[1].buf[r * mono[1].stride + c] >> 1) + 128;
}
// Now perform simple motion estimation and compensation
int nominal_block_width = nominal_block_size;
int nominal_block_height = nominal_block_size;
int block_width, block_height;
FILE *fp;
fopen_s(&fp, "Logdata.m", "w");
fprintf_s(fp, "myvec = [");
for (r = 0; r < height; r += block_height)
{
block_height = nominal_block_height;
if ((r + block_height) > height)
block_height = height - r;
for (c = 0; c < width; c += block_width)
{
block_width = nominal_block_width;
if ((c + block_width) > width)
block_width = width - c;
mvector vec = find_motion(&(ref_frame), &(mono[1]),
r, c, block_width, block_height, 4);
fprintf_s(fp, "%f,%f;\n",vec.x, vec.y);
motion_comp(&(ref_frame), &output, vec,
r, c, block_width, block_height, 4);
draw_line(&(mono[1]), &output_comps[1], vec,
r, c, block_width, block_height);
}
}
fprintf_s(fp, "];");
/*
for (int r = 0; r < height; ++r)
for (int c = 0; c < width; ++c) {
output_comps[0].buf[r * output_comps[0].stride + c] = (output.buf[r * output.stride + c]);
output_comps[1].buf[r * output_comps[1].stride + c] = (output.buf[r * output.stride + c]);
output_comps[2].buf[r * output_comps[2].stride + c] = (output.buf[r * output.stride + c]);
}
*/
float sum = 0;
for (int r = 0; r < height; ++r)
for (int c = 0; c < width; ++c) {
float diff = 0;
diff = mono[1].buf[r * mono[1].stride + c] - output.buf[r * output.stride + c];
diff *= diff;
sum += diff;
}
sum = sum / (width * height);
printf("The MSE is %f \r\n", sum);
// Write the motion compensated image out
bmp_out out;
if ((err_code = bmp_out__open(&out, argv[3], width, height, 3)) != 0)
throw err_code;
io_byte *out_line = new io_byte[width * 3];
for (r = height - 1; r >= 0; r--)
{ // "r" holds the true row index we are writing, since the image is
// written upside down in BMP files.
for (n = 0; n < 3; n++)
{
io_byte *dst = out_line + n; // Points to first sample of component n
int *src = output_comps[n].buf + r * output_comps[n].stride;
for (int c = 0; c < width; c++, dst += 3) {
if (src[c] > 255) {
src[c] = 255;
}
else if (src[c] < 0) {
src[c] = 0;
}
*dst = (io_byte)src[c];
}
}
bmp_out__put_line(&out, out_line);
}
bmp_out__close(&out);
delete[] line;
clock_t end_time = clock();
float elaps = ((float)(end_time - start_time)) / CLOCKS_PER_SEC;
printf_s("The runtime is %f seconds!\n\r", elaps);
system("Pause");
}
catch (int exc) {
if (exc == IO_ERR_NO_FILE)
fprintf(stderr, "Cannot open supplied input or output file.\n");
else if (exc == IO_ERR_FILE_HEADER)
fprintf(stderr, "Error encountered while parsing BMP file header.\n");
else if (exc == IO_ERR_UNSUPPORTED)
fprintf(stderr, "Input uses an unsupported BMP file format.\n Current "
"simple example supports only 8-bit and 24-bit data.\n");
else if (exc == IO_ERR_FILE_TRUNC)
fprintf(stderr, "Input or output file truncated unexpectedly.\n");
else if (exc == IO_ERR_FILE_NOT_OPEN)
fprintf(stderr, "Trying to access a file which is not open!(?)\n");
return -1;
}
return 0;
}
static int end_frame(AVFilterLink *link)
{
DeshakeContext *deshake = link->dst->priv;
AVFilterBufferRef *in = link->cur_buf;
AVFilterBufferRef *out = link->dst->outputs[0]->out_buf;
Transform t = {{0},0}, orig = {{0},0};
float matrix[9];
float alpha = 2.0 / deshake->refcount;
char tmp[256];
link->cur_buf = NULL; /* it is in 'in' now */
if (deshake->cx < 0 || deshake->cy < 0 || deshake->cw < 0 || deshake->ch < 0) {
// Find the most likely global motion for the current frame
find_motion(deshake, (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0], in->data[0], link->w, link->h, in->linesize[0], &t);
} else {
uint8_t *src1 = (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0];
uint8_t *src2 = in->data[0];
deshake->cx = FFMIN(deshake->cx, link->w);
deshake->cy = FFMIN(deshake->cy, link->h);
if ((unsigned)deshake->cx + (unsigned)deshake->cw > link->w) deshake->cw = link->w - deshake->cx;
if ((unsigned)deshake->cy + (unsigned)deshake->ch > link->h) deshake->ch = link->h - deshake->cy;
// Quadword align right margin
deshake->cw &= ~15;
src1 += deshake->cy * in->linesize[0] + deshake->cx;
src2 += deshake->cy * in->linesize[0] + deshake->cx;
find_motion(deshake, src1, src2, deshake->cw, deshake->ch, in->linesize[0], &t);
}
// Copy transform so we can output it later to compare to the smoothed value
orig.vector.x = t.vector.x;
orig.vector.y = t.vector.y;
orig.angle = t.angle;
orig.zoom = t.zoom;
// Generate a one-sided moving exponential average
deshake->avg.vector.x = alpha * t.vector.x + (1.0 - alpha) * deshake->avg.vector.x;
deshake->avg.vector.y = alpha * t.vector.y + (1.0 - alpha) * deshake->avg.vector.y;
deshake->avg.angle = alpha * t.angle + (1.0 - alpha) * deshake->avg.angle;
deshake->avg.zoom = alpha * t.zoom + (1.0 - alpha) * deshake->avg.zoom;
// Remove the average from the current motion to detect the motion that
// is not on purpose, just as jitter from bumping the camera
t.vector.x -= deshake->avg.vector.x;
t.vector.y -= deshake->avg.vector.y;
t.angle -= deshake->avg.angle;
t.zoom -= deshake->avg.zoom;
// Invert the motion to undo it
t.vector.x *= -1;
t.vector.y *= -1;
t.angle *= -1;
// Write statistics to file
if (deshake->fp) {
snprintf(tmp, 256, "%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f\n", orig.vector.x, deshake->avg.vector.x, t.vector.x, orig.vector.y, deshake->avg.vector.y, t.vector.y, orig.angle, deshake->avg.angle, t.angle, orig.zoom, deshake->avg.zoom, t.zoom);
fwrite(tmp, sizeof(char), strlen(tmp), deshake->fp);
}
// Turn relative current frame motion into absolute by adding it to the
// last absolute motion
t.vector.x += deshake->last.vector.x;
t.vector.y += deshake->last.vector.y;
t.angle += deshake->last.angle;
t.zoom += deshake->last.zoom;
// Shrink motion by 10% to keep things centered in the camera frame
t.vector.x *= 0.9;
t.vector.y *= 0.9;
t.angle *= 0.9;
// Store the last absolute motion information
deshake->last.vector.x = t.vector.x;
deshake->last.vector.y = t.vector.y;
deshake->last.angle = t.angle;
deshake->last.zoom = t.zoom;
// Generate a luma transformation matrix
avfilter_get_matrix(t.vector.x, t.vector.y, t.angle, 1.0 + t.zoom / 100.0, matrix);
// Transform the luma plane
avfilter_transform(in->data[0], out->data[0], in->linesize[0], out->linesize[0], link->w, link->h, matrix, INTERPOLATE_BILINEAR, deshake->edge);
// Generate a chroma transformation matrix
avfilter_get_matrix(t.vector.x / (link->w / CHROMA_WIDTH(link)), t.vector.y / (link->h / CHROMA_HEIGHT(link)), t.angle, 1.0 + t.zoom / 100.0, matrix);
// Transform the chroma planes
avfilter_transform(in->data[1], out->data[1], in->linesize[1], out->linesize[1], CHROMA_WIDTH(link), CHROMA_HEIGHT(link), matrix, INTERPOLATE_BILINEAR, deshake->edge);
avfilter_transform(in->data[2], out->data[2], in->linesize[2], out->linesize[2], CHROMA_WIDTH(link), CHROMA_HEIGHT(link), matrix, INTERPOLATE_BILINEAR, deshake->edge);
// Store the current frame as the reference frame for calculating the
// motion of the next frame
if (deshake->ref != NULL)
avfilter_unref_buffer(deshake->ref);
// Cleanup the old reference frame
deshake->ref = in;
// Draw the transformed frame information
ff_draw_slice(link->dst->outputs[0], 0, link->h, 1);
return ff_end_frame(link->dst->outputs[0]);
}
