void ccv_bbf_classifier_cascade_free(ccv_bbf_classifier_cascade_t* cascade)
{
int i;
for (i = 0; i < ENDORSE(cascade->count); ++i)
{
ccfree(cascade->stage_classifier[i].feature);
ccfree(cascade->stage_classifier[i].alpha);
}
ccfree(cascade->stage_classifier);
ccfree(cascade);
}
void ccv_array_free(ccv_array_t* array)
{
if (!ccv_cache_opt || !(array->type & CCV_REUSABLE) || array->sig == 0)
{
array->refcount = 0;
ccfree(array->data);
ccfree(array);
} else {
size_t size = sizeof(ccv_array_t) + array->size * array->rsize;
ccv_cache_put(&ccv_cache, array->sig, array, size, 1 /* type 1 */);
}
}
static void _ccv_dense_vector_expand(ccv_sparse_matrix_t* mat, ccv_dense_vector_t* vector)
{
if (vector->prime == -1)
return;
vector->prime++;
int new_length = CCV_GET_SPARSE_PRIME(vector->prime);
int cell_width = CCV_GET_DATA_TYPE_SIZE(mat->type) * CCV_GET_CHANNEL(mat->type);
int new_step = (new_length * cell_width + 3) & -4;
ccv_matrix_cell_t new_data;
new_data.u8 = (unsigned char*)ccmalloc(new_step + sizeof(int) * new_length);
int* new_indice = (int*)(new_data.u8 + new_step);
int i;
for (i = 0; i < new_length; i++)
new_indice[i] = -1;
for (i = 0; i < vector->length; i++)
if (vector->indice[i] != -1)
{
int index = vector->indice[i];
int h = (index * 33) % new_length, j = 0;
while (new_indice[(h + j * j) % new_length] != index && new_indice[(h + j * j) % new_length] != -1)
j++;
j = (h + j * j) % new_length;
new_indice[j] = index;
memcpy(new_data.u8 + j * cell_width, vector->data.u8 + i * cell_width, cell_width);
}
vector->length = new_length;
ccfree(vector->data.u8);
vector->data = new_data;
vector->indice = new_indice;
}
static void _ccv_sparse_matrix_expand(ccv_sparse_matrix_t* mat)
{
int length = CCV_GET_SPARSE_PRIME(mat->prime);
mat->prime++;
int new_length = CCV_GET_SPARSE_PRIME(mat->prime);
ccv_dense_vector_t* new_vector = (ccv_dense_vector_t*)ccmalloc(new_length * sizeof(ccv_dense_vector_t));
int i;
for (i = 0; i < new_length; i++)
{
new_vector[i].index = -1;
new_vector[i].length = 0;
new_vector[i].next = 0;
}
for (i = 0; i < length; i++)
if (mat->vector[i].index != -1)
{
int h = (mat->vector[i].index * 33) % new_length;
if (new_vector[h].length == 0)
{
memcpy(new_vector + h, mat->vector + i, sizeof(ccv_dense_vector_t));
new_vector[h].next = 0;
} else {
ccv_dense_vector_t* t = (ccv_dense_vector_t*)ccmalloc(sizeof(ccv_dense_vector_t));
memcpy(t, mat->vector + i, sizeof(ccv_dense_vector_t));
t->next = new_vector[h].next;
new_vector[h].next = t;
}
ccv_dense_vector_t* iter = mat->vector[i].next;
while (iter != 0)
{
ccv_dense_vector_t* iter_next = iter->next;
h = (iter->index * 33) % new_length;
if (new_vector[h].length == 0)
{
memcpy(new_vector + h, iter, sizeof(ccv_dense_vector_t));
new_vector[h].next = 0;
ccfree(iter);
} else {
iter->next = new_vector[h].next;
new_vector[h].next = iter;
}
iter = iter_next;
}
}
ccfree(mat->vector);
mat->vector = new_vector;
}
void ccv_matrix_free_immediately(ccv_matrix_t* mat)
{
int type = *(int*)mat;
assert(!(type & CCV_UNMANAGED));
if (type & CCV_MATRIX_DENSE)
{
ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
dmt->refcount = 0;
ccfree(dmt);
} else if (type & CCV_MATRIX_SPARSE) {
ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
int i;
for (i = 0; i < CCV_GET_SPARSE_PRIME(smt->prime); i++)
if (smt->vector[i].index != -1)
{
ccv_dense_vector_t* iter = &smt->vector[i];
ccfree(iter->data.u8);
iter = iter->next;
while (iter != 0)
{
ccv_dense_vector_t* iter_next = iter->next;
ccfree(iter->data.u8);
ccfree(iter);
iter = iter_next;
}
}
ccfree(smt->vector);
ccfree(smt);
} else if ((type & CCV_MATRIX_CSR) || (type & CCV_MATRIX_CSC)) {
ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
csm->refcount = 0;
ccfree(csm);
}
}
void ccv_matrix_free_immediately(ccv_matrix_t* mat)
{
int type = *(int*)mat;
assert(!(type & CCV_UNMANAGED));
if (type & CCV_MATRIX_DENSE)
{
ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
dmt->refcount = 0;
ccfree(dmt);
} else if (type & CCV_MATRIX_SPARSE) {
ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
int i;
for (i = 0; i < smt->size; i++)
if (smt->index[i].ifbit)
ccfree(smt->vector[i].data.u8);
ccfree(smt->vector);
ccfree(smt);
} else if ((type & CCV_MATRIX_CSR) || (type & CCV_MATRIX_CSC)) {
ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
csm->refcount = 0;
ccfree(csm);
}
}
void ccv_matrix_free(ccv_matrix_t* mat)
{
int type = *(int*)mat;
assert(!(type & CCV_UNMANAGED));
if (type & CCV_MATRIX_DENSE)
{
ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
dmt->refcount = 0;
if (!ccv_cache_opt || // e don't enable cache
!(dmt->type & CCV_REUSABLE) || // or this is not a reusable piece
dmt->sig == 0 || // or this doesn't have valid signature
(dmt->type & CCV_NO_DATA_ALLOC)) // or this matrix is allocated as header-only, therefore we cannot cache it
ccfree(dmt);
else {
assert(CCV_GET_DATA_TYPE(dmt->type) == CCV_8U ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_32S ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_32F ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_64S ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_64F);
size_t size = ccv_compute_dense_matrix_size(dmt->rows, dmt->cols, dmt->type);
ccv_cache_put(&ccv_cache, dmt->sig, dmt, size, 0 /* type 0 */);
}
} else if (type & CCV_MATRIX_SPARSE) {
ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
int i;
for (i = 0; i < CCV_GET_SPARSE_PRIME(smt->prime); i++)
if (smt->vector[i].index != -1)
{
ccv_dense_vector_t* iter = &smt->vector[i];
ccfree(iter->data.u8);
iter = iter->next;
while (iter != 0)
{
ccv_dense_vector_t* iter_next = iter->next;
ccfree(iter->data.u8);
ccfree(iter);
iter = iter_next;
}
}
ccfree(smt->vector);
ccfree(smt);
} else if ((type & CCV_MATRIX_CSR) || (type & CCV_MATRIX_CSC)) {
ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
csm->refcount = 0;
ccfree(csm);
}
}
void ccv_matrix_free(ccv_matrix_t* mat)
{
int type = *(int*)mat;
assert(!(type & CCV_UNMANAGED));
if (type & CCV_MATRIX_DENSE)
{
ccv_dense_matrix_t* dmt = (ccv_dense_matrix_t*)mat;
dmt->refcount = 0;
if (!ccv_cache_opt || // e don't enable cache
!(dmt->type & CCV_REUSABLE) || // or this is not a reusable piece
dmt->sig == 0 || // or this doesn't have valid signature
(dmt->type & CCV_NO_DATA_ALLOC)) // or this matrix is allocated as header-only, therefore we cannot cache it
ccfree(dmt);
else {
assert(CCV_GET_DATA_TYPE(dmt->type) == CCV_8U ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_32S ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_32F ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_64S ||
CCV_GET_DATA_TYPE(dmt->type) == CCV_64F);
size_t size = ccv_compute_dense_matrix_size(dmt->rows, dmt->cols, dmt->type);
ccv_cache_put(&ccv_cache, dmt->sig, dmt, size, 0 /* type 0 */);
}
} else if (type & CCV_MATRIX_SPARSE) {
ccv_sparse_matrix_t* smt = (ccv_sparse_matrix_t*)mat;
int i;
for (i = 0; i < smt->size; i++)
{
if (smt->index[i].ifbit > 1)
ccfree(smt->vector[i].index); // It is a union of index / data, can just free them.
}
ccfree(smt->index);
ccfree(smt->vector);
ccfree(smt);
} else if ((type & CCV_MATRIX_CSR) || (type & CCV_MATRIX_CSC)) {
ccv_compressed_sparse_matrix_t* csm = (ccv_compressed_sparse_matrix_t*)mat;
csm->refcount = 0;
ccfree(csm);
}
}
void ccv_array_free_immediately(ccv_array_t* array)
{
array->refcount = 0;
ccfree(array->data);
ccfree(array);
}
static int _ccv_nnc_conv_forw_sse2(const ccv_nnc_tensor_view_t* const a, const ccv_nnc_tensor_t* const w, const ccv_nnc_tensor_t* const bias, const ccv_nnc_hint_t hint, ccv_nnc_tensor_view_t* const b)
{
const int a_nd = ccv_nnc_tensor_nd(a->info.dim);
assert(a_nd == CCV_NNC_MAX_DIM + 1 || a_nd == CCV_NNC_MAX_DIM + 2);
const int* adim = (a_nd == CCV_NNC_MAX_DIM + 1) ? a->info.dim : a->info.dim + 1;
const int b_nd = ccv_nnc_tensor_nd(b->info.dim);
assert(b_nd == CCV_NNC_MAX_DIM + 1 || b_nd == CCV_NNC_MAX_DIM + 2);
const int* bdim = (b_nd == CCV_NNC_MAX_DIM + 1) ? b->info.dim : b->info.dim + 1;
const int* ainc = CCV_IS_TENSOR_VIEW(a) ? ((a_nd == CCV_NNC_MAX_DIM + 1) ? a->inc : a->inc + 1) : adim;
const int* binc = CCV_IS_TENSOR_VIEW(b) ? ((b_nd == CCV_NNC_MAX_DIM + 1) ? b->inc : b->inc + 1) : bdim;
assert(w->info.dim[0] % 4 == 0);
float* x4w = 0;
ccmemalign((void **)&x4w, 16, sizeof(float) * w->info.dim[3] * w->info.dim[2] * w->info.dim[1] * w->info.dim[0]);
if (!x4w)
return CCV_NNC_EXEC_OOM;
_ccv_nnc_x4w_sse2(w->data.f32, w->info.dim, x4w);
int jump_dim = w->info.dim[0] / 4;
// Do naive tail partition unroll
#define main_for(tail_block) \
parallel_for(k, jump_dim) { \
int c; \
const float* ap = a->data.f32; \
float* bp = b->data.f32 + k * 4; \
/* kernel weight for one dim. */ \
const float* const x4wp = x4w + k * 4 * w->info.dim[1] * w->info.dim[2] * w->info.dim[3]; \
const float biasval[4] __attribute__ ((__aligned__(16))) = { \
bias->data.f32[k * 4], \
bias->data.f32[k * 4 + 1], \
bias->data.f32[k * 4 + 2], \
bias->data.f32[k * 4 + 3] \
}; \
/* This block will be cause in each for-loop, therefore, you can use it to generate some temporary variables. */ \
int i[CCV_NNC_MAX_DIM]; \
int n[CCV_NNC_MAX_DIM]; \
int m[CCV_NNC_MAX_DIM]; \
int j[CCV_NNC_MAX_DIM]; \
for (i[0] = 0; i[0] < bdim[0]; i[0]++) \
{ \
SET_BORDER_OFFSET_SIZE_FOR(0, i, hint, w->info.dim + 1, adim, n, m); \
const float* wpu = x4wp + n[0] * w->info.dim[2] * w->info.dim[3] * 4; \
for (i[1] = 0; i[1] < bdim[1]; i[1]++) \
{ \
SET_BORDER_OFFSET_SIZE_FOR(1, i, hint, w->info.dim + 1, adim, n, m); \
__m128 v40 = _mm_load_ps(biasval); \
__m128 v41 = _mm_setzero_ps(); \
__m128 v42 = _mm_setzero_ps(); \
__m128 v43 = _mm_setzero_ps(); \
const float* wpz = wpu + n[1] * w->info.dim[3] * 4; \
const float* apz = ap + ccv_max(i[1] * hint.stride.dim[1] - hint.border.begin[1], 0) * ainc[2]; \
for (j[0] = 0; j[0] < m[0]; j[0]++) \
{ \
for (j[1] = 0; j[1] < m[1]; j[1]++) \
{ \
for (c = 0; c < adim[2] - 3; c += 4) \
{ \
__m128 apz4 = _mm_loadu_ps(apz + j[1] * ainc[2] + c); \
const float* const wpzu = wpz + (j[1] * w->info.dim[3] + c) * 4; \
__m128 w40 = _mm_loadu_ps(wpzu); \
__m128 w41 = _mm_loadu_ps(wpzu + 4); \
__m128 w42 = _mm_loadu_ps(wpzu + 8); \
__m128 w43 = _mm_loadu_ps(wpzu + 12); \
__m128 apz40 = _mm_shuffle_ps(apz4, apz4, 0x00); \
__m128 apz41 = _mm_shuffle_ps(apz4, apz4, 0x55); \
__m128 apz42 = _mm_shuffle_ps(apz4, apz4, 0xAA); \
__m128 apz43 = _mm_shuffle_ps(apz4, apz4, 0xFF); \
v40 =_mm_add_ps(_mm_mul_ps(w40, apz40), v40); \
v41 =_mm_add_ps(_mm_mul_ps(w41, apz41), v41); \
v42 =_mm_add_ps(_mm_mul_ps(w42, apz42), v42); \
v43 =_mm_add_ps(_mm_mul_ps(w43, apz43), v43); \
} \
tail_block /* insert executions for tail partition */ \
} \
wpz += w->info.dim[2] * w->info.dim[3] * 4; \
apz += ainc[1] * ainc[2]; \
} \
__m128 v4 = _mm_add_ps(_mm_add_ps(v40, v41), _mm_add_ps(v42, v43)); \
_mm_stream_ps(bp + i[1] * binc[2], v4); \
} \
bp += binc[1] * binc[2]; \
ap += ainc[1] * ainc[2] * (ccv_max((i[0] + 1) * hint.stride.dim[0] - hint.border.begin[0], 0) - ccv_max(i[0] * hint.stride.dim[0] - hint.border.begin[0], 0)); \
} \
} parallel_endfor
if (w->info.dim[3] % 4 == 0)
{
main_for();
} else if (w->info.dim[3] % 4 == 3) { // unroll the last for-loops
#define tail_block \
__m128 apz40 = _mm_load1_ps(apz + j[1] * ainc[2] + c); \
__m128 apz41 = _mm_load1_ps(apz + j[1] * ainc[2] + c + 1); \
__m128 apz42 = _mm_load1_ps(apz + j[1] * ainc[2] + c + 2); \
const float* const wpzu = wpz + (j[1] * w->info.dim[3] + c) * 4; \
__m128 w40 = _mm_loadu_ps(wpzu); \
__m128 w41 = _mm_loadu_ps(wpzu + 4); \
__m128 w42 = _mm_loadu_ps(wpzu + 8); \
v40 = _mm_add_ps(_mm_mul_ps(w40, apz40), v40); \
v41 = _mm_add_ps(_mm_mul_ps(w41, apz41), v41); \
v42 = _mm_add_ps(_mm_mul_ps(w42, apz42), v42);
main_for(tail_block);
#undef tail_block
} else if (w->info.dim[3] % 4 == 2) { // unroll the last for-loops
#define tail_block \
__m128 apz40 = _mm_load1_ps(apz + j[1] * ainc[2] + c); \
__m128 apz41 = _mm_load1_ps(apz + j[1] * ainc[2] + c + 1); \
const float* const wpzu = wpz + (j[1] * w->info.dim[3] + c) * 4; \
__m128 w40 = _mm_loadu_ps(wpzu); \
__m128 w41 = _mm_loadu_ps(wpzu + 4); \
v40 = _mm_add_ps(_mm_mul_ps(w40, apz40), v40); \
v41 = _mm_add_ps(_mm_mul_ps(w41, apz41), v41);
main_for(tail_block);
#undef tail_block
} else {
#define tail_block \
__m128 apz4 = _mm_load1_ps(apz + j[1] * ainc[2] + c); \
const float* const wpzu = wpz + (j[1] * w->info.dim[3] + c) * 4; \
__m128 w4 = _mm_loadu_ps(wpzu); \
v40 = _mm_add_ps(_mm_mul_ps(w4, apz4), v40);
main_for(tail_block);
#undef tail_block
}
#undef main_for
ccfree(x4w);
return CCV_NNC_EXEC_SUCCESS;
}
int main(int argc, char** argv)
{
#ifdef HAVE_AVCODEC
#ifdef HAVE_AVFORMAT
#ifdef HAVE_SWSCALE
assert(argc == 6);
ccv_rect_t box = ccv_rect(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), atoi(argv[5]));
box.width = box.width - box.x + 1;
box.height = box.height - box.y + 1;
printf("%d,%d,%d,%d,%f\n", box.x, box.y, box.width + box.x - 1, box.height + box.y - 1, 1.0f);
// init av-related structs
AVFormatContext* ic = 0;
int video_stream = -1;
AVStream* video_st = 0;
AVFrame* picture = 0;
AVFrame rgb_picture;
memset(&rgb_picture, 0, sizeof(AVPicture));
AVPacket packet;
memset(&packet, 0, sizeof(AVPacket));
av_init_packet(&packet);
av_register_all();
avformat_network_init();
// load video and codec
avformat_open_input(&ic, argv[1], 0, 0);
avformat_find_stream_info(ic, 0);
int i;
for (i = 0; i < ic->nb_streams; i++)
{
AVCodecContext* enc = ic->streams[i]->codec;
enc->thread_count = 2;
if (AVMEDIA_TYPE_VIDEO == enc->codec_type && video_stream < 0)
{
AVCodec* codec = avcodec_find_decoder(enc->codec_id);
if (!codec || avcodec_open2(enc, codec, 0) < 0)
continue;
video_stream = i;
video_st = ic->streams[i];
picture = avcodec_alloc_frame();
rgb_picture.data[0] = (uint8_t*)ccmalloc(avpicture_get_size(PIX_FMT_RGB24, enc->width, enc->height));
avpicture_fill((AVPicture*)&rgb_picture, rgb_picture.data[0], PIX_FMT_RGB24, enc->width, enc->height);
break;
}
}
int got_picture = 0;
while (!got_picture)
{
int result = av_read_frame(ic, &packet);
if (result == AVERROR(EAGAIN))
continue;
avcodec_decode_video2(video_st->codec, picture, &got_picture, &packet);
}
ccv_enable_default_cache();
struct SwsContext* picture_ctx = sws_getCachedContext(0, video_st->codec->width, video_st->codec->height, video_st->codec->pix_fmt, video_st->codec->width, video_st->codec->height, PIX_FMT_RGB24, SWS_BICUBIC, 0, 0, 0);
sws_scale(picture_ctx, (const uint8_t* const*)picture->data, picture->linesize, 0, video_st->codec->height, rgb_picture.data, rgb_picture.linesize);
ccv_dense_matrix_t* x = 0;
ccv_read(rgb_picture.data[0], &x, CCV_IO_RGB_RAW | CCV_IO_GRAY, video_st->codec->height, video_st->codec->width, rgb_picture.linesize[0]);
ccv_tld_t* tld = ccv_tld_new(x, box, ccv_tld_default_params);
ccv_dense_matrix_t* y = 0;
for (;;)
{
got_picture = 0;
int result = av_read_frame(ic, &packet);
if (result == AVERROR(EAGAIN))
continue;
avcodec_decode_video2(video_st->codec, picture, &got_picture, &packet);
if (!got_picture)
break;
sws_scale(picture_ctx, (const uint8_t* const*)picture->data, picture->linesize, 0, video_st->codec->height, rgb_picture.data, rgb_picture.linesize);
ccv_read(rgb_picture.data[0], &y, CCV_IO_RGB_RAW | CCV_IO_GRAY, video_st->codec->height, video_st->codec->width, rgb_picture.linesize[0]);
ccv_tld_info_t info;
ccv_comp_t newbox = ccv_tld_track_object(tld, x, y, &info);
printf("%04d: performed learn: %d, performed track: %d, successfully track: %d; %d passed fern detector, %d passed nnc detector, %d merged, %d confident matches, %d close matches\n", tld->count, info.perform_learn, info.perform_track, info.track_success, info.ferns_detects, info.nnc_detects, info.clustered_detects, info.confident_matches, info.close_matches);
ccv_dense_matrix_t* image = 0;
ccv_read(rgb_picture.data[0], &image, CCV_IO_RGB_RAW | CCV_IO_RGB_COLOR, video_st->codec->height, video_st->codec->width, rgb_picture.linesize[0]);
// draw out
// for (i = 0; i < tld->top->rnum; i++)
if (tld->found)
{
ccv_comp_t* comp = &newbox; // (ccv_comp_t*)ccv_array_get(tld->top, i);
if (comp->rect.x >= 0 && comp->rect.x + comp->rect.width < image->cols &&
comp->rect.y >= 0 && comp->rect.y + comp->rect.height < image->rows)
{
int x, y;
for (x = comp->rect.x; x < comp->rect.x + comp->rect.width; x++)
{
image->data.u8[image->step * comp->rect.y + x * 3] =
image->data.u8[image->step * (comp->rect.y + comp->rect.height - 1) + x * 3] = 255;
image->data.u8[image->step * comp->rect.y + x * 3 + 1] =
image->data.u8[image->step * (comp->rect.y + comp->rect.height - 1) + x * 3 + 1] =
image->data.u8[image->step * comp->rect.y + x * 3 + 2] =
image->data.u8[image->step * (comp->rect.y + comp->rect.height - 1) + x * 3 + 2] = 0;
}
for (y = comp->rect.y; y < comp->rect.y + comp->rect.height; y++)
{
image->data.u8[image->step * y + comp->rect.x * 3] =
image->data.u8[image->step * y + (comp->rect.x + comp->rect.width - 1) * 3] = 255;
image->data.u8[image->step * y + comp->rect.x * 3 + 1] =
image->data.u8[image->step * y + (comp->rect.x + comp->rect.width - 1) * 3 + 1] =
image->data.u8[image->step * y + comp->rect.x * 3 + 2] =
image->data.u8[image->step * y + (comp->rect.x + comp->rect.width - 1) * 3 + 2] = 0;
}
}
}
char filename[1024];
sprintf(filename, "tld-out/output-%04d.png", tld->count);
ccv_write(image, filename, 0, CCV_IO_PNG_FILE, 0);
ccv_matrix_free(image);
if (tld->found)
printf("%d,%d,%d,%d,%f\n", newbox.rect.x, newbox.rect.y, newbox.rect.width + newbox.rect.x - 1, newbox.rect.height + newbox.rect.y - 1, newbox.confidence);
else
printf("NaN,NaN,NaN,NaN,NaN\n");
x = y;
y = 0;
}
ccv_matrix_free(x);
ccv_tld_free(tld);
ccfree(rgb_picture.data[0]);
ccv_disable_cache();
#endif
#endif
#endif
return 0;
}
ccv_array_t* ccv_bbf_detect_objects(ccv_dense_matrix_t* a, ccv_bbf_classifier_cascade_t** _cascade, int count, ccv_bbf_param_t params)
{
int hr = a->rows / ENDORSE(params.size.height);
int wr = a->cols / ENDORSE(params.size.width);
double scale = pow(2., 1. / (params.interval + 1.));
APPROX int next = params.interval + 1;
int scale_upto = (int)(log((double)ccv_min(hr, wr)) / log(scale));
ccv_dense_matrix_t** pyr = (ccv_dense_matrix_t**)alloca(ENDORSE(scale_upto + next * 2) * 4 * sizeof(ccv_dense_matrix_t*));
memset(pyr, 0, (scale_upto + next * 2) * 4 * sizeof(ccv_dense_matrix_t*));
if (ENDORSE(params.size.height != _cascade[0]->size.height || params.size.width != _cascade[0]->size.width))
ccv_resample(a, &pyr[0], 0, a->rows * ENDORSE(_cascade[0]->size.height / params.size.height), a->cols * ENDORSE(_cascade[0]->size.width / params.size.width), CCV_INTER_AREA);
else
pyr[0] = a;
APPROX int i;
int j, k, t, x, y, q;
for (i = 1; ENDORSE(i < ccv_min(params.interval + 1, scale_upto + next * 2)); i++)
ccv_resample(pyr[0], &pyr[i * 4], 0, (int)(pyr[0]->rows / pow(scale, i)), (int)(pyr[0]->cols / pow(scale, i)), CCV_INTER_AREA);
for (i = next; ENDORSE(i < scale_upto + next * 2); i++)
ccv_sample_down(pyr[i * 4 - next * 4], &pyr[i * 4], 0, 0, 0);
if (params.accurate)
for (i = next * 2; ENDORSE(i < scale_upto + next * 2); i++)
{
ccv_sample_down(pyr[i * 4 - next * 4], &pyr[i * 4 + 1], 0, 1, 0);
ccv_sample_down(pyr[i * 4 - next * 4], &pyr[i * 4 + 2], 0, 0, 1);
ccv_sample_down(pyr[i * 4 - next * 4], &pyr[i * 4 + 3], 0, 1, 1);
}
ccv_array_t* idx_seq;
ccv_array_t* seq = ccv_array_new(sizeof(ccv_comp_t), 64, 0);
ccv_array_t* seq2 = ccv_array_new(sizeof(ccv_comp_t), 64, 0);
ccv_array_t* result_seq = ccv_array_new(sizeof(ccv_comp_t), 64, 0);
/* detect in multi scale */
for (t = 0; t < count; t++)
{
ccv_bbf_classifier_cascade_t* cascade = _cascade[t];
APPROX float scale_x = (float) params.size.width / (float) cascade->size.width;
APPROX float scale_y = (float) params.size.height / (float) cascade->size.height;
ccv_array_clear(seq);
for (i = 0; ENDORSE(i < scale_upto); i++)
{
APPROX int dx[] = {0, 1, 0, 1};
APPROX int dy[] = {0, 0, 1, 1};
APPROX int i_rows = pyr[i * 4 + next * 8]->rows - ENDORSE(cascade->size.height >> 2);
APPROX int steps[] = { pyr[i * 4]->step, pyr[i * 4 + next * 4]->step, pyr[i * 4 + next * 8]->step };
APPROX int i_cols = pyr[i * 4 + next * 8]->cols - ENDORSE(cascade->size.width >> 2);
int paddings[] = { pyr[i * 4]->step * 4 - i_cols * 4,
pyr[i * 4 + next * 4]->step * 2 - i_cols * 2,
pyr[i * 4 + next * 8]->step - i_cols };
for (q = 0; q < (params.accurate ? 4 : 1); q++)
{
APPROX unsigned char* u8[] = { pyr[i * 4]->data.u8 + dx[q] * 2 + dy[q] * pyr[i * 4]->step * 2, pyr[i * 4 + next * 4]->data.u8 + dx[q] + dy[q] * pyr[i * 4 + next * 4]->step, pyr[i * 4 + next * 8 + q]->data.u8 };
for (y = 0; ENDORSE(y < i_rows); y++)
{
for (x = 0; ENDORSE(x < i_cols); x++)
{
APPROX float sum;
APPROX int flag = 1;
ccv_bbf_stage_classifier_t* classifier = cascade->stage_classifier;
for (j = 0; j < ENDORSE(cascade->count); ++j, ++classifier)
{
sum = 0;
APPROX float* alpha = classifier->alpha;
ccv_bbf_feature_t* feature = classifier->feature;
for (k = 0; k < ENDORSE(classifier->count); ++k, alpha += 2, ++feature)
sum += alpha[_ccv_run_bbf_feature(feature, ENDORSE(steps), u8)];
if (ENDORSE(sum) < ENDORSE(classifier->threshold))
{
flag = 0;
break;
}
}
if (ENDORSE(flag))
{
ccv_comp_t comp;
comp.rect = ccv_rect((int)((x * 4 + dx[q] * 2) * scale_x + 0.5), (int)((y * 4 + dy[q] * 2) * scale_y + 0.5), (int)(cascade->size.width * scale_x + 0.5), (int)(cascade->size.height * scale_y + 0.5));
comp.neighbors = 1;
comp.classification.id = t;
comp.classification.confidence = sum;
ccv_array_push(seq, &comp);
}
u8[0] += 4;
u8[1] += 2;
u8[2] += 1;
}
u8[0] += paddings[0];
u8[1] += paddings[1];
u8[2] += paddings[2];
}
}
scale_x *= scale;
scale_y *= scale;
}
/* the following code from OpenCV's haar feature implementation */
if(params.min_neighbors == 0)
{
for (i = 0; ENDORSE(i < seq->rnum); i++)
{
ccv_comp_t* comp = (ccv_comp_t*)ENDORSE(ccv_array_get(seq, i));
ccv_array_push(result_seq, comp);
}
} else {
idx_seq = 0;
ccv_array_clear(seq2);
// group retrieved rectangles in order to filter out noise
int ncomp = ccv_array_group(seq, &idx_seq, _ccv_is_equal_same_class, 0);
ccv_comp_t* comps = (ccv_comp_t*)ccmalloc((ncomp + 1) * sizeof(ccv_comp_t));
memset(comps, 0, (ncomp + 1) * sizeof(ccv_comp_t));
// count number of neighbors
for(i = 0; ENDORSE(i < seq->rnum); i++)
{
ccv_comp_t r1 = *(ccv_comp_t*)ENDORSE(ccv_array_get(seq, i));
int idx = *(int*)ENDORSE(ccv_array_get(idx_seq, i));
if (ENDORSE(comps[idx].neighbors) == 0)
comps[idx].classification.confidence = r1.classification.confidence;
++comps[idx].neighbors;
comps[idx].rect.x += r1.rect.x;
comps[idx].rect.y += r1.rect.y;
comps[idx].rect.width += r1.rect.width;
comps[idx].rect.height += r1.rect.height;
comps[idx].classification.id = r1.classification.id;
comps[idx].classification.confidence = ccv_max(comps[idx].classification.confidence, r1.classification.confidence);
}
// calculate average bounding box
for(i = 0; ENDORSE(i < ncomp); i++)
{
int n = ENDORSE(comps[i].neighbors);
if(n >= params.min_neighbors)
{
ccv_comp_t comp;
comp.rect.x = (comps[i].rect.x * 2 + n) / (2 * n);
comp.rect.y = (comps[i].rect.y * 2 + n) / (2 * n);
comp.rect.width = (comps[i].rect.width * 2 + n) / (2 * n);
comp.rect.height = (comps[i].rect.height * 2 + n) / (2 * n);
comp.neighbors = comps[i].neighbors;
comp.classification.id = comps[i].classification.id;
comp.classification.confidence = comps[i].classification.confidence;
ccv_array_push(seq2, &comp);
}
}
// filter out small face rectangles inside large face rectangles
for(i = 0; ENDORSE(i < seq2->rnum); i++)
{
ccv_comp_t r1 = *(ccv_comp_t*)ENDORSE(ccv_array_get(seq2, i));
APPROX int flag = 1;
for(j = 0; ENDORSE(j < seq2->rnum); j++)
{
ccv_comp_t r2 = *(ccv_comp_t*)ENDORSE(ccv_array_get(seq2, j));
APPROX int distance = (int)(r2.rect.width * 0.25 + 0.5);
if(ENDORSE(i != j &&
r1.classification.id == r2.classification.id &&
r1.rect.x >= r2.rect.x - distance &&
r1.rect.y >= r2.rect.y - distance &&
r1.rect.x + r1.rect.width <= r2.rect.x + r2.rect.width + distance &&
r1.rect.y + r1.rect.height <= r2.rect.y + r2.rect.height + distance &&
(r2.neighbors > ccv_max(3, r1.neighbors) || r1.neighbors < 3)))
{
flag = 0;
break;
}
}
if(ENDORSE(flag))
ccv_array_push(result_seq, &r1);
}
ccv_array_free(idx_seq);
ccfree(comps);
}
}
ccv_array_free(seq);
ccv_array_free(seq2);
ccv_array_t* result_seq2;
/* the following code from OpenCV's haar feature implementation */
if (params.flags & CCV_BBF_NO_NESTED)
{
result_seq2 = ccv_array_new(sizeof(ccv_comp_t), 64, 0);
idx_seq = 0;
// group retrieved rectangles in order to filter out noise
int ncomp = ccv_array_group(result_seq, &idx_seq, _ccv_is_equal, 0);
ccv_comp_t* comps = (ccv_comp_t*)ccmalloc((ncomp + 1) * sizeof(ccv_comp_t));
memset(comps, 0, (ncomp + 1) * sizeof(ccv_comp_t));
// count number of neighbors
for(i = 0; ENDORSE(i < result_seq->rnum); i++)
{
ccv_comp_t r1 = *(ccv_comp_t*)ENDORSE(ccv_array_get(result_seq, i));
int idx = *(int*)ENDORSE(ccv_array_get(idx_seq, i));
if (ENDORSE(comps[idx].neighbors == 0 || comps[idx].classification.confidence < r1.classification.confidence))
{
comps[idx].classification.confidence = r1.classification.confidence;
comps[idx].neighbors = 1;
comps[idx].rect = r1.rect;
comps[idx].classification.id = r1.classification.id;
}
}
// calculate average bounding box
for(i = 0; ENDORSE(i < ncomp); i++)
if(ENDORSE(comps[i].neighbors))
ccv_array_push(result_seq2, &comps[i]);
ccv_array_free(result_seq);
ccfree(comps);
} else {
result_seq2 = result_seq;
}
for (i = 1; ENDORSE(i < scale_upto + next * 2); i++)
ccv_matrix_free(pyr[i * 4]);
if (params.accurate)
for (i = next * 2; ENDORSE(i < scale_upto + next * 2); i++)
{
ccv_matrix_free(pyr[i * 4 + 1]);
ccv_matrix_free(pyr[i * 4 + 2]);
ccv_matrix_free(pyr[i * 4 + 3]);
}
if (ENDORSE(params.size.height != _cascade[0]->size.height || params.size.width != _cascade[0]->size.width))
ccv_matrix_free(pyr[0]);
return result_seq2;
}
// compute harmonic mean of precision / recall of swt
static void _ccv_evaluate_wolf(ccv_array_t* words, ccv_array_t* truth, ccv_swt_param_t params, double* precision, double* recall)
{
if (words->rnum == 0 || truth->rnum == 0)
return;
int j, k;
double total_recall = 0, total_precision = 0;
int* cG = (int*)ccmalloc(sizeof(int) * truth->rnum);
int* cD = (int*)ccmalloc(sizeof(int) * words->rnum);
memset(cG, 0, sizeof(int) * truth->rnum);
memset(cD, 0, sizeof(int) * words->rnum);
double* mG = (double*)ccmalloc(sizeof(double) * truth->rnum * words->rnum);
double* mD = (double*)ccmalloc(sizeof(double) * truth->rnum * words->rnum);
memset(mG, 0, sizeof(double) * truth->rnum * words->rnum);
memset(mD, 0, sizeof(double) * truth->rnum * words->rnum);
for (j = 0; j < truth->rnum; j++)
{
ccv_rect_t* rect = (ccv_rect_t*)ccv_array_get(truth, j);
for (k = 0; k < words->rnum; k++)
{
ccv_rect_t* target = (ccv_rect_t*)ccv_array_get(words, k);
int match = ccv_max(ccv_min(target->x + target->width, rect->x + rect->width) - ccv_max(target->x, rect->x), 0) * ccv_max(ccv_min(target->y + target->height, rect->y + rect->height) - ccv_max(target->y, rect->y), 0);
if (match > 0)
{
mG[j * words->rnum + k] = (double)match / (double)(rect->width * rect->height);
mD[k * truth->rnum + j] = (double)match / (double)(target->width * target->height);
++cG[j];
++cD[k];
}
}
}
unsigned char* tG = (unsigned char*)ccmalloc(truth->rnum);
unsigned char* tD = (unsigned char*)ccmalloc(words->rnum);
memset(tG, 0, truth->rnum);
memset(tD, 0, words->rnum);
// one to one match
for (j = 0; j < truth->rnum; j++)
{
if (cG[j] != 1)
continue;
ccv_rect_t* rect = (ccv_rect_t*)ccv_array_get(truth, j);
for (k = 0; k < words->rnum; k++)
{
if (cD[k] != 1)
continue;
ccv_rect_t* target = (ccv_rect_t*)ccv_array_get(words, k);
if (mG[j * words->rnum + k] >= one_g && mD[k * truth->rnum + j] >= one_d)
{
double dx = (target->x + target->width * 0.5) - (rect->x + rect->width * 0.5);
double dy = (target->y + target->height * 0.5) - (rect->y + rect->height * 0.5);
double d = sqrt(dx * dx + dy * dy) * 2.0 / (sqrt(target->width * target->width + target->height * target->height) + sqrt(rect->width * rect->width + rect->height * rect->height));
if (d < center_diff_thr)
{
total_recall += 1.0;
total_precision += 1.0;
assert(tG[j] == 0);
assert(tD[k] == 0);
tG[j] = tD[k] = 1;
}
}
}
}
int* many = (int*)ccmalloc(sizeof(int) * ccv_max(words->rnum, truth->rnum));
// one to many match, starts with ground truth
for (j = 0; j < truth->rnum; j++)
{
if (tG[j] || cG[j] <= 1)
continue;
double one_sum = 0;
int no_many = 0;
for (k = 0; k < words->rnum; k++)
{
if (tD[k])
continue;
double many_single = mD[k * truth->rnum + j];
if (many_single >= one_d)
{
one_sum += mG[j * words->rnum + k];
many[no_many] = k;
++no_many;
}
}
if (no_many == 1)
{
// degrade to one to one match
if (mG[j * words->rnum + many[0]] >= one_g && mD[many[0] * truth->rnum + j] >= one_d)
{
total_recall += 1.0;
total_precision += 1.0;
tG[j] = tD[many[0]] = 1;
}
} else if (one_sum >= one_g) {
for (k = 0; k < no_many; k++)
tD[many[k]] = 1;
total_recall += om_one;
total_precision += om_one / (1 + log(no_many));
}
}
// one to many match, with estimate
for (k = 0; k < words->rnum; k++)
{
if (tD[k] || cD[k] <= 1)
continue;
double one_sum = 0;
int no_many = 0;
for (j = 0; j < truth->rnum; j++)
{
if (tG[j])
continue;
double many_single = mG[j * words->rnum + k];
if (many_single >= one_g)
{
one_sum += mD[k * truth->rnum + j];
many[no_many] = j;
++no_many;
}
}
if (no_many == 1)
{
// degrade to one to one match
if (mG[many[0] * words->rnum + k] >= one_g && mD[k * truth->rnum + many[0]] >= one_d)
{
total_recall += 1.0;
total_precision += 1.0;
tG[many[0]] = tD[k] = 1;
}
} else if (one_sum >= one_g) {
for (j = 0; j < no_many; j++)
tG[many[j]] = 1;
total_recall += om_one / (1 + log(no_many));
total_precision += om_one;
}
}
ccfree(many);
ccfree(tG);
ccfree(tD);
ccfree(cG);
ccfree(cD);
ccfree(mG);
ccfree(mD);
assert(total_precision < words->rnum + 0.1);
assert(total_recall < truth->rnum + 0.1);
if (precision)
*precision = total_precision;
if (recall)
*recall = total_recall;
}
/* it is a supposely cleaner and faster implementation than original OpenCV (ccv_canny_deprecated,
* removed, since the newer implementation achieve bit accuracy with OpenCV's), after a lot
* profiling, the current implementation still uses integer to speed up */
void ccv_canny(ccv_dense_matrix_t* a, ccv_dense_matrix_t** b, int type, int size, double low_thresh, double high_thresh)
{
assert(a->type & CCV_C1);
ccv_declare_matrix_signature(sig, a->sig != 0, ccv_sign_with_format(64, "ccv_canny(%d,%lf,%lf)", size, low_thresh, high_thresh), a->sig, 0);
type = (type == 0) ? CCV_8U | CCV_C1 : CCV_GET_DATA_TYPE(type) | CCV_C1;
ccv_dense_matrix_t* db = *b = ccv_dense_matrix_renew(*b, a->rows, a->cols, CCV_C1 | CCV_ALL_DATA_TYPE, type, sig);
ccv_matrix_return_if_cached(, db);
if ((a->type & CCV_8U) || (a->type & CCV_32S))
{
ccv_dense_matrix_t* dx = 0;
ccv_dense_matrix_t* dy = 0;
ccv_sobel(a, &dx, 0, size, 0);
ccv_sobel(a, &dy, 0, 0, size);
/* special case, all integer */
int low = (int)(low_thresh + 0.5);
int high = (int)(high_thresh + 0.5);
int* dxi = dx->data.i32;
int* dyi = dy->data.i32;
int i, j;
int* mbuf = (int*)alloca(3 * (a->cols + 2) * sizeof(int));
memset(mbuf, 0, 3 * (a->cols + 2) * sizeof(int));
int* rows[3];
rows[0] = mbuf + 1;
rows[1] = mbuf + (a->cols + 2) + 1;
rows[2] = mbuf + 2 * (a->cols + 2) + 1;
for (j = 0; j < a->cols; j++)
rows[1][j] = abs(dxi[j]) + abs(dyi[j]);
dxi += a->cols;
dyi += a->cols;
int* map = (int*)ccmalloc(sizeof(int) * (a->rows + 2) * (a->cols + 2));
memset(map, 0, sizeof(int) * (a->cols + 2));
int* map_ptr = map + a->cols + 2 + 1;
int map_cols = a->cols + 2;
int** stack = (int**)ccmalloc(sizeof(int*) * a->rows * a->cols);
int** stack_top = stack;
int** stack_bottom = stack;
for (i = 1; i <= a->rows; i++)
{
/* the if clause should be unswitched automatically, no need to manually do so */
if (i == a->rows)
memset(rows[2], 0, sizeof(int) * a->cols);
else
for (j = 0; j < a->cols; j++)
rows[2][j] = abs(dxi[j]) + abs(dyi[j]);
int* _dx = dxi - a->cols;
int* _dy = dyi - a->cols;
map_ptr[-1] = 0;
int suppress = 0;
for (j = 0; j < a->cols; j++)
{
int f = rows[1][j];
if (f > low)
{
int x = abs(_dx[j]);
int y = abs(_dy[j]);
int s = _dx[j] ^ _dy[j];
/* x * tan(22.5) */
int tg22x = x * (int)(0.4142135623730950488016887242097 * (1 << 15) + 0.5);
/* x * tan(67.5) == 2 * x + x * tan(22.5) */
int tg67x = tg22x + ((x + x) << 15);
y <<= 15;
/* it is a little different from the Canny original paper because we adopted the coordinate system of
* top-left corner as origin. Thus, the derivative of y convolved with matrix:
* |-1 -2 -1|
* | 0 0 0|
* | 1 2 1|
* actually is the reverse of real y. Thus, the computed angle will be mirrored around x-axis.
* In this case, when angle is -45 (135), we compare with north-east and south-west, and for 45,
* we compare with north-west and south-east (in traditional coordinate system sense, the same if we
* adopt top-left corner as origin for "north", "south", "east", "west" accordingly) */
#define high_block \
{ \
if (f > high && !suppress && map_ptr[j - map_cols] != 2) \
{ \
map_ptr[j] = 2; \
suppress = 1; \
*(stack_top++) = map_ptr + j; \
} else { \
map_ptr[j] = 1; \
} \
continue; \
}
/* sometimes, we end up with same f in integer domain, for that case, we will take the first occurrence
* suppressing the second with flag */
if (y < tg22x)
{
if (f > rows[1][j - 1] && f >= rows[1][j + 1])
high_block;
} else if (y > tg67x) {
if (f > rows[0][j] && f >= rows[2][j])
high_block;
} else {
s = s < 0 ? -1 : 1;
if (f > rows[0][j - s] && f > rows[2][j + s])
high_block;
}
#undef high_block
}
map_ptr[j] = 0;
suppress = 0;
}
map_ptr[a->cols] = 0;
map_ptr += map_cols;
dxi += a->cols;
dyi += a->cols;
int* row = rows[0];
rows[0] = rows[1];
rows[1] = rows[2];
rows[2] = row;
}
memset(map_ptr - map_cols - 1, 0, sizeof(int) * (a->cols + 2));
int dr[] = {-1, 1, -map_cols - 1, -map_cols, -map_cols + 1, map_cols - 1, map_cols, map_cols + 1};
while (stack_top > stack_bottom)
{
map_ptr = *(--stack_top);
for (i = 0; i < 8; i++)
if (map_ptr[dr[i]] == 1)
{
map_ptr[dr[i]] = 2;
*(stack_top++) = map_ptr + dr[i];
}
}
map_ptr = map + map_cols + 1;
unsigned char* b_ptr = db->data.u8;
#define for_block(_, _for_set) \
for (i = 0; i < a->rows; i++) \
{ \
for (j = 0; j < a->cols; j++) \
_for_set(b_ptr, j, (map_ptr[j] == 2), 0); \
map_ptr += map_cols; \
b_ptr += db->step; \
}
ccv_matrix_setter(db->type, for_block);
#undef for_block
ccfree(stack);
ccfree(map);
ccv_matrix_free(dx);
ccv_matrix_free(dy);
} else {
/* general case, use all ccv facilities to deal with it */
ccv_dense_matrix_t* mg = 0;
ccv_dense_matrix_t* ag = 0;
ccv_gradient(a, &ag, 0, &mg, 0, size, size);
ccv_matrix_free(ag);
ccv_matrix_free(mg);
/* FIXME: Canny implementation for general case */
}
}