Gesture* DataProcesser::analyze(DataAnalyzer<Angles> data, int new_samples) { Gesture* gb; if (find_beat(data, &gb)) return gb; else if (find_motion(data, &gb)) return gb; else return nullptr; }
static int filter_frame(AVFilterLink *link, AVFrame *in) { DeshakeContext *deshake = link->dst->priv; AVFilterLink *outlink = link->dst->outputs[0]; AVFrame *out; Transform t = {{0},0}, orig = {{0},0}; float matrix_y[9], matrix_uv[9]; float alpha = 2.0 / deshake->refcount; char tmp[256]; int ret = 0; out = ff_get_video_buffer(outlink, outlink->w, outlink->h); if (!out) { av_frame_free(&in); return AVERROR(ENOMEM); } av_frame_copy_props(out, in); if (CONFIG_OPENCL && deshake->opencl) { ret = ff_opencl_deshake_process_inout_buf(link->dst,in, out); if (ret < 0) return ret; } if (deshake->cx < 0 || deshake->cy < 0 || deshake->cw < 0 || deshake->ch < 0) { // Find the most likely global motion for the current frame find_motion(deshake, (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0], in->data[0], link->w, link->h, in->linesize[0], &t); } else { uint8_t *src1 = (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0]; uint8_t *src2 = in->data[0]; deshake->cx = FFMIN(deshake->cx, link->w); deshake->cy = FFMIN(deshake->cy, link->h); if ((unsigned)deshake->cx + (unsigned)deshake->cw > link->w) deshake->cw = link->w - deshake->cx; if ((unsigned)deshake->cy + (unsigned)deshake->ch > link->h) deshake->ch = link->h - deshake->cy; // Quadword align right margin deshake->cw &= ~15; src1 += deshake->cy * in->linesize[0] + deshake->cx; src2 += deshake->cy * in->linesize[0] + deshake->cx; find_motion(deshake, src1, src2, deshake->cw, deshake->ch, in->linesize[0], &t); } // Copy transform so we can output it later to compare to the smoothed value orig.vector.x = t.vector.x; orig.vector.y = t.vector.y; orig.angle = t.angle; orig.zoom = t.zoom; // Generate a one-sided moving exponential average deshake->avg.vector.x = alpha * t.vector.x + (1.0 - alpha) * deshake->avg.vector.x; deshake->avg.vector.y = alpha * t.vector.y + (1.0 - alpha) * deshake->avg.vector.y; deshake->avg.angle = alpha * t.angle + (1.0 - alpha) * deshake->avg.angle; deshake->avg.zoom = alpha * t.zoom + (1.0 - alpha) * deshake->avg.zoom; // Remove the average from the current motion to detect the motion that // is not on purpose, just as jitter from bumping the camera t.vector.x -= deshake->avg.vector.x; t.vector.y -= deshake->avg.vector.y; t.angle -= deshake->avg.angle; t.zoom -= deshake->avg.zoom; // Invert the motion to undo it t.vector.x *= -1; t.vector.y *= -1; t.angle *= -1; // Write statistics to file if (deshake->fp) { snprintf(tmp, 256, "%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f\n", orig.vector.x, deshake->avg.vector.x, t.vector.x, orig.vector.y, deshake->avg.vector.y, t.vector.y, orig.angle, deshake->avg.angle, t.angle, orig.zoom, deshake->avg.zoom, t.zoom); fwrite(tmp, sizeof(char), strlen(tmp), deshake->fp); } // Turn relative current frame motion into absolute by adding it to the // last absolute motion t.vector.x += deshake->last.vector.x; t.vector.y += deshake->last.vector.y; t.angle += deshake->last.angle; t.zoom += deshake->last.zoom; // Shrink motion by 10% to keep things centered in the camera frame t.vector.x *= 0.9; t.vector.y *= 0.9; t.angle *= 0.9; // Store the last absolute motion information deshake->last.vector.x = t.vector.x; deshake->last.vector.y = t.vector.y; deshake->last.angle = t.angle; deshake->last.zoom = t.zoom; // Generate a luma transformation matrix avfilter_get_matrix(t.vector.x, t.vector.y, t.angle, 1.0 + t.zoom / 100.0, matrix_y); // Generate a chroma transformation matrix avfilter_get_matrix(t.vector.x / (link->w / CHROMA_WIDTH(link)), t.vector.y / (link->h / CHROMA_HEIGHT(link)), t.angle, 1.0 + t.zoom / 100.0, matrix_uv); // Transform the luma and chroma planes ret = deshake->transform(link->dst, link->w, link->h, CHROMA_WIDTH(link), CHROMA_HEIGHT(link), matrix_y, matrix_uv, INTERPOLATE_BILINEAR, deshake->edge, in, out); // Cleanup the old reference frame av_frame_free(&deshake->ref); if (ret < 0) return ret; // Store the current frame as the reference frame for calculating the // motion of the next frame deshake->ref = in; return ff_filter_frame(outlink, out); }
int main(int argc, char *argv[]) { if (argc != 4) { fprintf(stderr, "Usage: %s <bmp frame 1> <bmp frame 2> <bmp MC out>\n", argv[0]); return -1; } clock_t start_time = clock(); int err_code = 0; try { // Read the input image bmp_in in[2]; if ((err_code = bmp_in__open(&in[0], argv[1])) != 0) throw err_code; if ((err_code = bmp_in__open(&in[1], argv[2])) != 0) throw err_code; int width = in[0].cols, height = in[0].rows; if ((width != in[1].cols) || (height != in[1].rows)) { fprintf(stderr, "The two input frames have different dimensions.\n"); return -1; } my_image_comp mono[2]; mono[0].init(height, width, 4); // Leave a border of 4 (in case needed) mono[1].init(height, width, 4); // Leave a border of 4 (in case needed) mono[0].perform_boundary_extension(); int n, r, c; int num_comps = in[0].num_components; io_byte *line = new io_byte[width*num_comps]; for (n = 0; n < 2; n++) { for (r = height - 1; r >= 0; r--) { // "r" holds the true row index we are reading, since the image // is stored upside down in the BMP file. if ((err_code = bmp_in__get_line(&(in[n]), line)) != 0) throw err_code; io_byte *src = line; // Points to first sample of component n int *dst = mono[n].buf + r * mono[n].stride; for (c = 0; c < width; c++, src += num_comps) dst[c] = *src; } bmp_in__close(&(in[n])); } // Allocate storage for the motion compensated output my_image_comp output; output.init(height, width, 0); // Don't need a border for output my_image_comp inter,ref_frame; inter.init(height * 2, width * 2, 4); ref_frame.init(height * 4, width * 4, 4); interpolation(&mono[0], &inter); interpolation(&inter, &ref_frame); my_image_comp output_comps[3]; for (int i = 0; i < 3; ++i) { output_comps[i].init(height, width, 0); } for (int r = 0; r < height; ++r) for (int c = 0; c < width; ++c) { output_comps[0].buf[r * output_comps[0].stride + c] = (mono[1].buf[r * mono[1].stride + c] >> 1) + 128; output_comps[1].buf[r * output_comps[1].stride + c] = (mono[1].buf[r * mono[1].stride + c] >> 1) + 128; output_comps[2].buf[r * output_comps[2].stride + c] = (mono[1].buf[r * mono[1].stride + c] >> 1) + 128; } // Now perform simple motion estimation and compensation int nominal_block_width = nominal_block_size; int nominal_block_height = nominal_block_size; int block_width, block_height; FILE *fp; fopen_s(&fp, "Logdata.m", "w"); fprintf_s(fp, "myvec = ["); for (r = 0; r < height; r += block_height) { block_height = nominal_block_height; if ((r + block_height) > height) block_height = height - r; for (c = 0; c < width; c += block_width) { block_width = nominal_block_width; if ((c + block_width) > width) block_width = width - c; mvector vec = find_motion(&(ref_frame), &(mono[1]), r, c, block_width, block_height, 4); fprintf_s(fp, "%f,%f;\n",vec.x, vec.y); motion_comp(&(ref_frame), &output, vec, r, c, block_width, block_height, 4); draw_line(&(mono[1]), &output_comps[1], vec, r, c, block_width, block_height); } } fprintf_s(fp, "];"); /* for (int r = 0; r < height; ++r) for (int c = 0; c < width; ++c) { output_comps[0].buf[r * output_comps[0].stride + c] = (output.buf[r * output.stride + c]); output_comps[1].buf[r * output_comps[1].stride + c] = (output.buf[r * output.stride + c]); output_comps[2].buf[r * output_comps[2].stride + c] = (output.buf[r * output.stride + c]); } */ float sum = 0; for (int r = 0; r < height; ++r) for (int c = 0; c < width; ++c) { float diff = 0; diff = mono[1].buf[r * mono[1].stride + c] - output.buf[r * output.stride + c]; diff *= diff; sum += diff; } sum = sum / (width * height); printf("The MSE is %f \r\n", sum); // Write the motion compensated image out bmp_out out; if ((err_code = bmp_out__open(&out, argv[3], width, height, 3)) != 0) throw err_code; io_byte *out_line = new io_byte[width * 3]; for (r = height - 1; r >= 0; r--) { // "r" holds the true row index we are writing, since the image is // written upside down in BMP files. for (n = 0; n < 3; n++) { io_byte *dst = out_line + n; // Points to first sample of component n int *src = output_comps[n].buf + r * output_comps[n].stride; for (int c = 0; c < width; c++, dst += 3) { if (src[c] > 255) { src[c] = 255; } else if (src[c] < 0) { src[c] = 0; } *dst = (io_byte)src[c]; } } bmp_out__put_line(&out, out_line); } bmp_out__close(&out); delete[] line; clock_t end_time = clock(); float elaps = ((float)(end_time - start_time)) / CLOCKS_PER_SEC; printf_s("The runtime is %f seconds!\n\r", elaps); system("Pause"); } catch (int exc) { if (exc == IO_ERR_NO_FILE) fprintf(stderr, "Cannot open supplied input or output file.\n"); else if (exc == IO_ERR_FILE_HEADER) fprintf(stderr, "Error encountered while parsing BMP file header.\n"); else if (exc == IO_ERR_UNSUPPORTED) fprintf(stderr, "Input uses an unsupported BMP file format.\n Current " "simple example supports only 8-bit and 24-bit data.\n"); else if (exc == IO_ERR_FILE_TRUNC) fprintf(stderr, "Input or output file truncated unexpectedly.\n"); else if (exc == IO_ERR_FILE_NOT_OPEN) fprintf(stderr, "Trying to access a file which is not open!(?)\n"); return -1; } return 0; }
static int end_frame(AVFilterLink *link) { DeshakeContext *deshake = link->dst->priv; AVFilterBufferRef *in = link->cur_buf; AVFilterBufferRef *out = link->dst->outputs[0]->out_buf; Transform t = {{0},0}, orig = {{0},0}; float matrix[9]; float alpha = 2.0 / deshake->refcount; char tmp[256]; link->cur_buf = NULL; /* it is in 'in' now */ if (deshake->cx < 0 || deshake->cy < 0 || deshake->cw < 0 || deshake->ch < 0) { // Find the most likely global motion for the current frame find_motion(deshake, (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0], in->data[0], link->w, link->h, in->linesize[0], &t); } else { uint8_t *src1 = (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0]; uint8_t *src2 = in->data[0]; deshake->cx = FFMIN(deshake->cx, link->w); deshake->cy = FFMIN(deshake->cy, link->h); if ((unsigned)deshake->cx + (unsigned)deshake->cw > link->w) deshake->cw = link->w - deshake->cx; if ((unsigned)deshake->cy + (unsigned)deshake->ch > link->h) deshake->ch = link->h - deshake->cy; // Quadword align right margin deshake->cw &= ~15; src1 += deshake->cy * in->linesize[0] + deshake->cx; src2 += deshake->cy * in->linesize[0] + deshake->cx; find_motion(deshake, src1, src2, deshake->cw, deshake->ch, in->linesize[0], &t); } // Copy transform so we can output it later to compare to the smoothed value orig.vector.x = t.vector.x; orig.vector.y = t.vector.y; orig.angle = t.angle; orig.zoom = t.zoom; // Generate a one-sided moving exponential average deshake->avg.vector.x = alpha * t.vector.x + (1.0 - alpha) * deshake->avg.vector.x; deshake->avg.vector.y = alpha * t.vector.y + (1.0 - alpha) * deshake->avg.vector.y; deshake->avg.angle = alpha * t.angle + (1.0 - alpha) * deshake->avg.angle; deshake->avg.zoom = alpha * t.zoom + (1.0 - alpha) * deshake->avg.zoom; // Remove the average from the current motion to detect the motion that // is not on purpose, just as jitter from bumping the camera t.vector.x -= deshake->avg.vector.x; t.vector.y -= deshake->avg.vector.y; t.angle -= deshake->avg.angle; t.zoom -= deshake->avg.zoom; // Invert the motion to undo it t.vector.x *= -1; t.vector.y *= -1; t.angle *= -1; // Write statistics to file if (deshake->fp) { snprintf(tmp, 256, "%f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f\n", orig.vector.x, deshake->avg.vector.x, t.vector.x, orig.vector.y, deshake->avg.vector.y, t.vector.y, orig.angle, deshake->avg.angle, t.angle, orig.zoom, deshake->avg.zoom, t.zoom); fwrite(tmp, sizeof(char), strlen(tmp), deshake->fp); } // Turn relative current frame motion into absolute by adding it to the // last absolute motion t.vector.x += deshake->last.vector.x; t.vector.y += deshake->last.vector.y; t.angle += deshake->last.angle; t.zoom += deshake->last.zoom; // Shrink motion by 10% to keep things centered in the camera frame t.vector.x *= 0.9; t.vector.y *= 0.9; t.angle *= 0.9; // Store the last absolute motion information deshake->last.vector.x = t.vector.x; deshake->last.vector.y = t.vector.y; deshake->last.angle = t.angle; deshake->last.zoom = t.zoom; // Generate a luma transformation matrix avfilter_get_matrix(t.vector.x, t.vector.y, t.angle, 1.0 + t.zoom / 100.0, matrix); // Transform the luma plane avfilter_transform(in->data[0], out->data[0], in->linesize[0], out->linesize[0], link->w, link->h, matrix, INTERPOLATE_BILINEAR, deshake->edge); // Generate a chroma transformation matrix avfilter_get_matrix(t.vector.x / (link->w / CHROMA_WIDTH(link)), t.vector.y / (link->h / CHROMA_HEIGHT(link)), t.angle, 1.0 + t.zoom / 100.0, matrix); // Transform the chroma planes avfilter_transform(in->data[1], out->data[1], in->linesize[1], out->linesize[1], CHROMA_WIDTH(link), CHROMA_HEIGHT(link), matrix, INTERPOLATE_BILINEAR, deshake->edge); avfilter_transform(in->data[2], out->data[2], in->linesize[2], out->linesize[2], CHROMA_WIDTH(link), CHROMA_HEIGHT(link), matrix, INTERPOLATE_BILINEAR, deshake->edge); // Store the current frame as the reference frame for calculating the // motion of the next frame if (deshake->ref != NULL) avfilter_unref_buffer(deshake->ref); // Cleanup the old reference frame deshake->ref = in; // Draw the transformed frame information ff_draw_slice(link->dst->outputs[0], 0, link->h, 1); return ff_end_frame(link->dst->outputs[0]); }