esvmOutput *esvmSIME(esvmParameters *params, IplImage *img, esvmModel *model)
{
//userTasks needs to be greater than 4.
//This is for performance. less than 4 threads doesn't make sense!
//Also I think (there was an assumption made in some version
//, I forget if it is still there or not.)
//otherwise binning histograms (binHists)
//will not work properly.
//this can be fixed, but I haven't done it.
assert(params->userTasks >= 4);
//computing hog pyramid and reading whogs can be made parallel
#ifdef ESVM_PERFORMANCE_COUNTERS
double hogTime = CycleTimer::currentSeconds();
#endif
esvmHogPyr *hogpyr = computeHogScale(img,params->cellWidth,params->maxHogLevels,params->minHogDim,
params->levelsPerOctave,params->minImageScale,params->hogEnablePadding,
params->hogPadding,params->userTasks,params->useMexResize);
#ifdef ESVM_PERFORMANCE_COUNTERS
hogTime -= CycleTimer::currentSeconds();
#endif
const esvmHogPyr *whogpyr = model->hogpyr;
const esvmHog **whogs = (const esvmHog **) whogpyr->hogs;
const int numWeights = whogpyr->num;
const float *bWeight = model->b;
#ifdef ESVM_PERFORMANCE_COUNTERS
double convTime = CycleTimer::currentSeconds();
#endif
esvmArr2_f *convResults = convolvePyramids(hogpyr,whogpyr,params->convEnablePadding,
params->userTasks);
#ifdef ESVM_PERFORMANCE_COUNTERS
convTime -= CycleTimer::currentSeconds();
#endif
//allocate memory for bounding boxes per exemplar
esvmBoxes *boxesArr = (esvmBoxes *)esvmCalloc(numWeights*hogpyr->num,sizeof(esvmBoxes));
assert(params->maxTotalBoxesPerExemplar > params->maxWindowsPerExemplar);
for(int w=0;w<numWeights;w++) {
boxesArr[w].arr = (float *)esvmMalloc(params->maxTotalBoxesPerExemplar*ESVM_BOX_DIM*sizeof(float));
boxesArr[w].num = 0;
std::fill(boxesArr[w].arr,boxesArr[w].arr+params->maxTotalBoxesPerExemplar*ESVM_BOX_DIM,
ESVM_FLOAT_MIN);
}
float *maxers = (float *)esvmMalloc(numWeights*sizeof(float));
std::fill(maxers,maxers+numWeights,ESVM_FLOAT_MIN);
float *negScores = (float *)esvmMalloc(params->maxTotalBoxesPerExemplar*sizeof(float));
float *topScores = (float *)esvmMalloc(params->maxTotalBoxesPerExemplar*sizeof(float));
int *topInds = (int *)esvmMalloc(params->maxTotalBoxesPerExemplar*sizeof(int));
float *topBoxes = (float *)esvmMalloc(params->maxTotalBoxesPerExemplar*ESVM_BOX_DIM*sizeof(float));
//parallel loop.
int numBoxes = 0;
#ifdef ESVM_PERFORMANCE_COUNTERS
double nmsTime = CycleTimer::currentSeconds();
#endif
//serial loop because maxers are maintained from higher levels!
for(int i=hogpyr->num-1;i>=0;i--) {
//serial loop
for(int w=0;w<numWeights;w++) {
esvmArr2_f *convOut = &(convResults[i*numWeights+w]);
subtractScalar(convOut->arr,convOut->rows,convOut->cols,bWeight[w]);
int nkeep;
//hogPadding is subtracted from the indices.
float detectionThreshold = max(maxers[w],params->detectionThreshold);
//float detectionThreshold = params->detectionThreshold;
int *indices = sort2DIndex(convOut->arr,convOut->rows,convOut->cols,
ESVM_DESCEND_SORT,ESVM_THRESHOLD,detectionThreshold,&nkeep,
params->hogPadding);
if(nkeep==0) {
continue;
}
//arrays for top-k sorting
const int topK = min(boxesArr[w].num+nkeep,params->maxWindowsPerExemplar);
//concatenate current boxes to the boxes already detected by exemplar
float *bboxes = &(boxesArr[w].arr[boxesArr[w].num*ESVM_BOX_DIM]);
int *tmpIndex = indices;
int dim1 = convOut->rows*convOut->cols;
float resizing = params->cellWidth/hogpyr->scale[i];
//get the bounding boxes in the original image
//need to rescale
assert(boxesArr[w].num+nkeep <= params->maxTotalBoxesPerExemplar);
//printf("NKEEP is %d\n",nkeep);
for(int j=0;j<nkeep;j++) {
float *bboxL = bboxes+j*ESVM_BOX_DIM;
ARR_RMIN_P(bboxL) = ((*(tmpIndex+j))*resizing);
ARR_CMIN_P(bboxL) = ((*(tmpIndex+j+dim1))*resizing);
ARR_RMAX_P(bboxL) = ((*(tmpIndex+j)+whogs[w]->rows)*resizing)-1;
ARR_CMAX_P(bboxL) = ((*(tmpIndex+j+dim1)+whogs[w]->cols)*resizing)-1;
//Put negative of score inside.
//This is useful for finding top-k elements
negScores[boxesArr[w].num+j] = -convOut->arr[j];
ARR_SCORE_P(bboxL) = convOut->arr[j];
ARR_SCALE_P(bboxL) = hogpyr->scale[i];
ARR_CLASS_P(bboxL) = (int)whogs[w]->classId;
ARR_EXID_P(bboxL) = w;
}
//find top-k boxes
psort(negScores, boxesArr[w].num+nkeep, topK, topScores, topInds);
int *tmpTop = topInds;
bboxes = boxesArr[w].arr;
for(int j=0;j<topK;j++) {
float *bboxL = topBoxes+j*ESVM_BOX_DIM;
float *bboxR = bboxes+topInds[j]*ESVM_BOX_DIM;
ARR_COPY_P(bboxR,bboxL);
negScores[j] = -ARR_SCORE_P(bboxR);
}
//now copy back the current boxes into the list of boxes for this exemplar
memcpy(boxesArr[w].arr,topBoxes,topK*ESVM_BOX_DIM*sizeof(float));
boxesArr[w].num = topK;
if(topK >= params->maxWindowsPerExemplar) {
//update maxers if topK > threshold
maxers[w] = -topScores[topK-1];
}
free(indices);
free(convOut->arr);
}
}
//more cleanup
free(convResults);
free(maxers);
free(negScores);
free(topScores);
free(topInds);
free(topBoxes);
//perform nms on each exemplar's boxes
esvmBoxes *nmsBoxesArr = (esvmBoxes *)esvmCalloc(numWeights*hogpyr->num,sizeof(esvmBoxes));
int totalBoxes = 0;
for(int w=0;w<numWeights;w++) {
nms(boxesArr[w].arr,boxesArr[w].num, params->nmsOverlapThreshold,
&(nmsBoxesArr[w].num),&(nmsBoxesArr[w].arr));
totalBoxes += (nmsBoxesArr[w].num);
free(boxesArr[w].arr);
}
free(boxesArr);
#ifdef ESVM_PERFORMANCE_COUNTERS
nmsTime -= CycleTimer::currentSeconds();
#endif
//assign output
esvmOutput *output = (esvmOutput *)esvmMalloc(sizeof(esvmOutput));
//collect all the boxes together
output->boxes = (esvmBoxes *)esvmCalloc(1,sizeof(esvmBoxes));
output->boxes->num = totalBoxes;
if(totalBoxes>0)
output->boxes->arr = (float *)esvmMalloc(totalBoxes*ESVM_BOX_DIM*sizeof(float));
int count = 0;
for(int w=0;w<numWeights;w++) {
for(int j=0;j<nmsBoxesArr[w].num;j++) {
ARR_COPY(nmsBoxesArr[w].arr,j,output->boxes->arr,count);
count++;
}
free(nmsBoxesArr[w].arr);
}
free(nmsBoxesArr);
if(params->saveHogPyr==false) {
freeHogPyramid(hogpyr);
output->hogpyr = (esvmHogPyr *)esvmMalloc(sizeof(esvmHogPyr));
output->hogpyr->num = 0;
}
else {
output->hogpyr = hogpyr;
}
#ifdef ESVM_PERFORMANCE_COUNTERS
output->perf.hogTime = -hogTime*1000;
output->perf.convTime = -convTime*1000;
output->perf.nmsTime = -nmsTime*1000;
#endif
return output;
}
void bus_detector_cnn::findBus(string inputImagePath)
{
Mat inputImage = imread(inputImagePath);
//resize(inputImage,inputImage,Size(640,480));
//Making a multiscale input image
vector<float> processScale = {1.0,0.8,0.6,0.4,0.2};
Mat scaledImage;
vector<busBBox> allBusBoxes;
for(uint scaleNo = 0 ; scaleNo < processScale.size() ; scaleNo++)
{
resize(inputImage,scaledImage,Size(),processScale[scaleNo],processScale[scaleNo]);
//Suppress scale if images is smaller than CNN input size
if(scaledImage.rows >= windowSizeH && scaledImage.cols >= windowSizeW)
{
vector<Rect> scaleBboxes;
Mat rowWindowImage; //sliding windows container
slidingWindows(scaledImage,windowStrides,processScale[scaleNo],rowWindowImage,scaleBboxes);
//#ifdef DEBUGF
cout<<"Scale : "<<processScale[scaleNo]<<" Row Image Size: "<<rowWindowImage.rows<<" x "<<rowWindowImage.cols<<" Total windows:"<<scaleBboxes.size()<<endl;
//#endif // DEBUGF
int64 t0 = cv::getTickCount();
float *windows_h = (float*) rowWindowImage.data; //pointer to sliding window in row-fashion format (NO-OP)
float *windows_d;
checkCudaErrors( cudaMalloc(&windows_d,scaleBboxes.size() * windowSizeH * windowSizeW * 3 *sizeof(float)));
checkCudaErrors( cudaMemcpy(windows_d,windows_h,scaleBboxes.size() * 3 * windowSizeH * windowSizeW * sizeof(float),cudaMemcpyHostToDevice));
uint nBatch = scaleBboxes.size() / concurrentWindows;
float *output_d;
float *output_h = new float[totalClass*scaleBboxes.size()];
float *batchwindows_d;
for(uint batch = 0 ; batch < nBatch ; batch++)
{
int n = concurrentWindows; //input data to cuDNN in this project is in NCHW format (Nimages-channels-row-col)
int c = 3;
int h = windowSizeH;
int w = windowSizeW;
batchwindows_d = &windows_d[batch * (concurrentWindows * c *h * w)];
//Start Detector
busDetector.feed_forward(&batchwindows_d,n,c,h,w,&output_d);
//Copy back result (Be note that copy in block-format is faster than copy only desired elements)
checkCudaErrors( cudaMemcpy(&output_h[batch * concurrentWindows * c],output_d,n * c * h * w * sizeof(float),cudaMemcpyDeviceToHost));
checkCudaErrors( cudaFree(output_d));
}
int remainingWindows = scaleBboxes.size() - (concurrentWindows * nBatch);
if (remainingWindows > 0)
{
int n = remainingWindows; //input data to cuDNN in this project is in NCHW format (Nimages-channels-row-col)
int c = 3;
int h = windowSizeH;
int w = windowSizeW;
batchwindows_d = &windows_d[nBatch * (concurrentWindows * c *h * w)];
//Start Detector
busDetector.feed_forward(&batchwindows_d,n,c,h,w,&output_d);
//Copy back result (Be note that copy in block-format is faster than copy only desired elements)
checkCudaErrors( cudaMemcpy(&output_h[nBatch * concurrentWindows * c],output_d,n * c * h * w * sizeof(float),cudaMemcpyDeviceToHost));
checkCudaErrors( cudaFree(output_d));
}
int64 t1 = cv::getTickCount();
double secs = (t1-t0)/cv::getTickFrequency();
cout<<"GPU Time: "<<secs<<endl;
checkCudaErrors( cudaFree(windows_d));
pickBusBBoxes(scaleBboxes,output_h,allBusBoxes);
delete [] output_h;
}
//inputImage too small
else
{
break;
}
}
//Non-correct version of nms (???)
nms(allBusBoxes,0.4);
//Display only high score box
for(uint box = 0 ; box < allBusBoxes.size() ; box++)
{
if(allBusBoxes[box].keep && allBusBoxes[box].score > 0.6)
{
cout<<"["<<allBusBoxes[box].bbox.x<<" "<<allBusBoxes[box].bbox.y<<" "<<allBusBoxes[box].bbox.width<<" "<<allBusBoxes[box].bbox.height<<"] Score: "<<allBusBoxes[box].score<<endl;
rectangle(inputImage,allBusBoxes[box].bbox,Scalar(255,255,255),1);
}
}
imshow("Image",inputImage);
imwrite("/home/thananop/1_out.jpg",inputImage);
waitKey(0);
}
bool Type::operator>=(const Type& t) const {
NodeManagerScope nms(d_nodeManager);
return *d_typeNode >= *t.d_typeNode;
}
bool Type::isSubtypeOf(Type t) const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isSubtypeOf(*t.d_typeNode);
}
Type Type::getBaseType() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->getBaseType().toType();
}
Cardinality Type::getCardinality() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->getCardinality();
}
bool Type::isWellFounded() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isWellFounded();
}
/**
* Is this a predicate type? NOTE: all predicate types are also
* function types.
*/
bool Type::isPredicate() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isPredicate();
}
/** Is this an array type? */
bool Type::isArray() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isArray();
}
/** Is this a datatype type? */
bool Type::isDatatype() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isDatatype() || d_typeNode->isParametricDatatype();
}
/** Is this a function type? */
bool Type::isFunction() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isFunction();
}
/** Is this the bit-vector type? */
bool Type::isBitVector() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isBitVector();
}
string Type::toString() const {
NodeManagerScope nms(d_nodeManager);
stringstream ss;
ss << *d_typeNode;
return ss.str();
}
void Type::toStream(std::ostream& out) const {
NodeManagerScope nms(d_nodeManager);
out << *d_typeNode;
return;
}
Type::~Type() {
NodeManagerScope nms(d_nodeManager);
delete d_typeNode;
}
ostream& operator<<(ostream& out, const Type& t) {
NodeManagerScope nms(t.d_nodeManager);
return out << *Type::getTypeNode(t);
}
size_t DatatypeType::getArity() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->getNumChildren() - 1;
}
/** Cast to a sort type */
bool Type::isSortConstructor() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isSortConstructor();
}
SubrangeBounds SubrangeType::getSubrangeBounds() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->getSubrangeBounds();
}
/** Is this an integer subrange */
bool Type::isSubrange() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isSubrange();
}
Expr Type::mkGroundTerm() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->mkGroundTerm().toExpr();
}
Type FunctionType::getRangeType() const {
NodeManagerScope nms(d_nodeManager);
CheckArgument(isNull() || isFunction(), this);
return makeType(d_typeNode->getRangeType());
}
bool Type::isComparableTo(Type t) const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->isComparableTo(*t.d_typeNode);
}
const Record& RecordType::getRecord() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->getRecord();
}
long detection_fprop(
float* conf, //score for each class for each box, num_box * num_class * bs
float* loc, //location for each box, box * 4 * bs
float* res_detection, //final memory restoring boxes, bs * top_k
float* prior_boxes, //num_boxes * 4
long * res_batch_len, //record count of result for each batch, bs
const long num_boxes, //num_boxes, each is a potential object
const long num_class, //number of class
const long bs, //batch size
const long nms_topk, //first top k box for nms result for each class
const long image_topk, //first top k box for input image
const float score_threshold, //threshold for accepting as a object for box
const float nms_threshold) //threshold for two overlapped boxes, too overlapped is one object
{
//sorted result of index
long* index_batch = malloc(bs*num_boxes*num_class*sizeof(long));
//scores to be sorted
float* scores_batch = malloc(bs*num_boxes*num_class*sizeof(float));
//temp result detections for each batch, grow when iterating among classes
float* temp_res_detection_batch = malloc(bs*num_class*nms_topk*6*sizeof(float));
//internal memory to restore sorted boxes for each class
float* internal_detection_batch = malloc(bs*nms_topk*5*sizeof(float));
//internal memory to restore transformed location
float* proposal_batch = malloc(bs*num_boxes*4*sizeof(float));
//transpose KLN to NKL
float* conf_t = malloc(num_boxes * num_class * bs * sizeof(float));
float* loc_t = malloc(num_boxes * 4* bs * sizeof(float));
mkl_somatcopy('r', 't', num_boxes*num_class, bs, 1.0, conf, bs, conf_t, num_boxes*num_class);
mkl_somatcopy('r', 't', num_boxes*4, bs, 1.0, loc, bs, loc_t, num_boxes*4);
//loop for batch size
#pragma omp parallel for
for(long b=0; b<bs; ++b) //loop for batch
{
float* scores = scores_batch + b * num_boxes*num_class;
float* temp_res_detection = temp_res_detection_batch + b * num_class*nms_topk*6;
long* index = index_batch + b * num_boxes*num_class;
float* internal_detection = internal_detection_batch + b * nms_topk*5;
float* proposal = proposal_batch + b * num_boxes*4;
//calculate class scores for this batch using softmax
float* conf_batch = conf_t + b * num_boxes * num_class;
softmax(conf_batch, num_boxes, num_class);
//store scores in an array
mkl_somatcopy('r', 't', num_boxes, num_class, 1.0, conf_batch, num_class, scores, num_boxes);
//transform locations in proposal
bbox_transform_inv(prior_boxes, loc_t + b * 4 * num_boxes, proposal, num_boxes);
long res_len = 0; //count of feasible boxes for this image
for(long c=1; c<num_class; ++c) //loop for classes
{
//for each class, sort out first nms_topk boxes, store result in index
long sort_nums_res = get_top_N_index(scores + c*num_boxes, nms_topk,
num_boxes, score_threshold, index);
//store location and score for the sorted results
if(sort_nums_res > 0)
{
//store location and score in internal_detection for overlapped check
for(long i=0; i<sort_nums_res; ++i)
{
for(long j=0; j<4; ++j)
internal_detection[i*5+j] = proposal[index[i]*4+j];
internal_detection[i*5+4] = scores[c*num_boxes+i];
}
//remove overlapped box
sort_nums_res = nms(internal_detection, index, nms_threshold, 1, sort_nums_res);
//store result in temp memory and add class number, thus width is 6
for(long i=0; i<sort_nums_res; ++i)
{
float* temp = temp_res_detection + (res_len+i)*6;
for(long j=0; j<5; ++j)
{
temp[j] = internal_detection[index[i]*5+j];
}
//add class number
temp[5] = c;
}
res_len += sort_nums_res;
}
}
//sort out first top_k boxes for this image
for(long i=0; i<res_len; ++i)
{
scores[i] = temp_res_detection[i*6+4];
index[i] = i;
}
long sort_nums_res = res_len;
if(sort_nums_res>image_topk) //sort first top_k out of res_len
{
sort_nums_res = get_top_N_index(scores, image_topk, res_len, 0.0, index);
}
//store sorted result in final output
float* temp = res_detection + b * image_topk * 6;
for(long i=0; i<sort_nums_res; ++i)
{
for(long j=0; j<6; ++j)
{
temp[i*6+j] = temp_res_detection[index[i]*6+j];
}
}
res_batch_len[b] = sort_nums_res;
}
free(conf_t);
free(loc_t);
free(index_batch);
free(scores_batch);
free(temp_res_detection_batch);
free(proposal_batch);
free(internal_detection_batch);
}
string SortConstructorType::getName() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->getAttribute(expr::VarNameAttr());
}
//_____________________________________________________________________________
void ProofTests::SlaveBegin(TTree * /*tree*/)
{
// The SlaveBegin() function is called after the Begin() function.
// When running with PROOF SlaveBegin() is called on each slave server.
// The tree argument is deprecated (on PROOF 0 is passed).
TString option = GetOption();
// Fill relevant members
ParseInput();
// Output histo
fStat = new TH1I("TestStat", "Test results", 20, .5, 20.5);
fOutput->Add(fStat);
// We were started
fStat->Fill(1.);
// Depends on the test
if (fTestType == 0) {
// Retrieve objects from the input list and copy them to the output
// H1
TList *h1list = dynamic_cast<TList*>(fInput->FindObject("h1list"));
if (h1list) {
// Retrieve objects from the input list and copy them to the output
TH1F *h1 = dynamic_cast<TH1F*>(h1list->FindObject("h1data"));
if (h1) {
TParameter<Double_t> *h1avg = dynamic_cast<TParameter<Double_t>*>(h1list->FindObject("h1avg"));
TParameter<Double_t> *h1rms = dynamic_cast<TParameter<Double_t>*>(h1list->FindObject("h1rms"));
if (h1avg && h1rms) {
if (TMath::Abs(h1avg->GetVal() - h1->GetMean()) < 0.0001) {
if (TMath::Abs(h1rms->GetVal() - h1->GetRMS()) < 0.0001) {
fStat->Fill(2.);
}
}
} else {
Info("SlaveBegin", "%d: info 'h1avg' or 'h1rms' not found!", fTestType);
}
} else {
Info("SlaveBegin", "%d: input histo 'h1data' not found!", fTestType);
}
} else {
Info("SlaveBegin", "%d: input list 'h1list' not found!", fTestType);
}
// H2
TList *h2list = dynamic_cast<TList*>(fInput->FindObject("h2list"));
if (h2list) {
// Retrieve objects from the input list and copy them to the output
TH1F *h2 = dynamic_cast<TH1F*>(h2list->FindObject("h2data"));
if (h2) {
TParameter<Double_t> *h2avg = dynamic_cast<TParameter<Double_t>*>(h2list->FindObject("h2avg"));
TParameter<Double_t> *h2rms = dynamic_cast<TParameter<Double_t>*>(h2list->FindObject("h2rms"));
if (h2avg && h2rms) {
if (TMath::Abs(h2avg->GetVal() - h2->GetMean()) < 0.0001) {
if (TMath::Abs(h2rms->GetVal() - h2->GetRMS()) < 0.0001) {
fStat->Fill(3.);
}
}
} else {
Info("SlaveBegin", "%d: info 'h2avg' or 'h2rms' not found!", fTestType);
}
} else {
Info("SlaveBegin", "%d: input histo 'h2data' not found!", fTestType);
}
} else {
Info("SlaveBegin", "%d: input list 'h2list' not found!", fTestType);
}
TNamed *iob = dynamic_cast<TNamed*>(fInput->FindObject("InputObject"));
if (iob) {
fStat->Fill(4.);
} else {
Info("SlaveBegin", "%d: input histo 'InputObject' not found!", fTestType);
}
} else if (fTestType == 1) {
// We should find in the input list the name of a test variable which should exist
// at this point in the gEnv table
TNamed *nm = dynamic_cast<TNamed*>(fInput->FindObject("testenv"));
if (nm) {
if (gEnv->Lookup(nm->GetTitle())) fStat->Fill(2.);
} else {
Info("SlaveBegin", "%d: TNamed with the test env info not found!", fTestType);
}
} else if (fTestType == 2) {
// We should find in the input list the list of names of test variables which should exist
// at this point in the gEnv table
TNamed *nm = dynamic_cast<TNamed*>(fInput->FindObject("testenv"));
if (nm) {
TString nms(nm->GetTitle()), nam;
Int_t from = 0;
while (nms.Tokenize(nam, from, ",")) {
if (gEnv->Lookup(nam)) {
Double_t xx = gEnv->GetValue(nam, -1.);
if (xx > 1.) fStat->Fill(xx);
}
}
} else {
Info("SlaveBegin", "%d: TNamed with the test env info not found!", fTestType);
}
}
}
size_t SortConstructorType::getArity() const {
NodeManagerScope nms(d_nodeManager);
return d_typeNode->getAttribute(expr::SortArityAttr());
}
Type Type::substitute(const Type& type, const Type& replacement) const {
NodeManagerScope nms(d_nodeManager);
return makeType(d_typeNode->substitute(*type.d_typeNode,
*replacement.d_typeNode));
}
int main(void){
// minimum size(pixels) of detection object (multiple of 12)
const int MinFaceSize = 72;
// thresholds
const float Threshold_12Layer = .5;
const float Threshold_24Layer = .01;
const float Threshold_48Layer = -.01;
const float Threshold_12CalibrationLayer = .1;
const float Threshold_24CalibrationLayer = .1;
const float Threshold_48CalibrationLayer = .1;
const float Threshold_12NMS = .3f;
const float Threshold_24NMS = .3f;
const float Threshold_48NMS = 1.0f;
// detection windows
struct Windows window[500];
// loop counter
int i, j, k;
int row, col;
int counter = 0; // detection window counter
// image information
int height, width, step, channels;
uchar *data, *data24, *data48;
// size, x, y for calibration
float *out_12c, *out_24c, *out_48c; // vector carrying s,x,y
float s, x, y;
int cali_x, cali_y, cali_w, cali_h;
// scores of the 12 layer
float res_12Layer;
float res_24Layer;
float res_48Layer;
// window sliding stride
const int Stride = 4;
// image pyramid rate
int pyr_rate = MinFaceSize / 12;
// image pyrimid stopping
bool flagStop = false;
// file path
char file[150];
strcpy(file, FILE_PATH);
strcat(file, TEST_IMAGE);
// alloc memory for 12x12 image
float **img = malloc(12 * sizeof(float*));
for (i = 0; i < 12; i++){
img[i] = malloc(12 * sizeof(float));
}
// alloc memory for 24x24 image
float **img24 = malloc(24 * sizeof(float*));
for (i = 0; i < 24; i++){
img24[i] = malloc(24 * sizeof(float));
}
// alloc memory for 48x48 image
float **img48 = malloc(48 * sizeof(float*));
for (i = 0; i < 48; i++){
img48[i] = malloc(48 * sizeof(float));
}
// for printing scores
CvFont font;
cvInitFont(&font, CV_FONT_HERSHEY_SIMPLEX, 0.3, 0.3, 0, 1, 8);
char word[5];
// load image
IplImage *srcImg, *dstImg;
srcImg = cvLoadImage(file, CV_LOAD_IMAGE_GRAYSCALE);
// original size of the image
const int WIDTH = srcImg->width;
const int HEIGHT = srcImg->height;
IplImage *originImg = cvCloneImage(srcImg);
IplImage *originImg1 = cvCloneImage(srcImg);
IplImage *originImg2 = cvCloneImage(srcImg);
IplImage *originImg3 = cvCloneImage(srcImg);
IplImage *input24Img = cvCreateImage(cvSize(24, 24), IPL_DEPTH_8U, 1);
IplImage *input48Img = cvCreateImage(cvSize(48, 48), IPL_DEPTH_8U, 1);
if (!srcImg){
printf("Could not load image file: %s\n", file);
exit(1);
}
// image pyramid loop starts
while (!flagStop){
counter = 0;
// image pyramid down
dstImg = doPyrDown(srcImg, pyr_rate);
// get the image data
width = dstImg -> width;
height = dstImg -> height;
step = dstImg -> widthStep;
channels = dstImg -> nChannels;
data = (uchar*)dstImg -> imageData;
IplImage *detectedImg_12Layer = cvCloneImage(dstImg);
// window sliding loop starts
for (row = 0; row + 12 <= height; row += Stride){
for (col = 0; col + 12 <= width; col += Stride){
// 12 layer, 12 calibration, NMS
preprocess(img, data, row, col, step, channels, 12);
res_12Layer = Layer12(img, 12, 12, channels);
// 12 layer passed
if (res_12Layer > Threshold_12Layer){
// 12 calibration layer
out_12c = CaliLayer12(img, 12, 12, channels, Threshold_12CalibrationLayer);
s = out_12c[0]; x = out_12c[1]; y = out_12c[2];
free(out_12c); // memory allocated in CaliLayer12
// ignoring returned NAN values(comparison involving them are always false)
if (s != s || x != x || y != y) continue;
// calibration
cali_x = col * pyr_rate - x * 12 * pyr_rate / s;
cali_y = row * pyr_rate - y * 12 * pyr_rate / s;
cali_w = 12 * pyr_rate / s;
cali_h = 12 * pyr_rate / s;
// make sure the calibrated window not beyond the image boundary
if (cali_x >= WIDTH || cali_y >= HEIGHT) continue;
cali_x = max(cali_x, 0);
cali_y = max(cali_y, 0);
cali_w = min(cali_w, WIDTH - cali_x);
cali_h = min(cali_h, HEIGHT - cali_y);
window[counter].x1 = cali_x; // x1
window[counter].y1 = cali_y; // y1
window[counter].x2 = cali_x + cali_w; // x2
window[counter].y2 = cali_y + cali_h; // y2
window[counter].score = res_12Layer; // 12 layer score
window[counter].iou = 0.0; // iou ratio
window[counter].dropped= false; // if it's dropped
counter++;
// end of 12 layer, 12 calibration
}
}
}
// window sliding loop ends
// sort the detection windows by score in descending order
mergeSort(window, 0, counter);
// display sorted windows surviving 12 layer
cvNamedWindow("12 layer", CV_WINDOW_AUTOSIZE);
for (i = 0; i < counter; i++){
cvRectangle(originImg, cvPoint(window[i].x1, window[i].y1), cvPoint(window[i].x2, window[i].y2), cvScalar(255,0,0,0), 2, 4, 0);
// printf("[#%d] x1: %d, y1: %d, x2: %d, y2: %d, score: %f, iou: %f, dropped: %s\n", i, window[i].x1, window[i].y1, window[i].x2, window[i].y2, window[i].score, window[i].iou, window[i].dropped ? "true" : "false");
if (window[i].dropped == false){
sprintf(word, "%.2f", window[i].score);
cvPutText(originImg, word, cvPoint(window[i].x1, window[i].y1), &font, cvScalar(255, 255, 255, 0));
}
}
cvShowImage("12 layer", originImg);
cvMoveWindow("12 layer", 10, 10);
printf("12 layer: x1: %d, y1: %d, x2: %d, y2: %d\n", window[15].x1, window[15].y1, window[15].x2, window[15].y2);
// NMS after 12 calibration
nms(window, counter, Threshold_12NMS);
// display sorted windows surviving 12 layer
cvNamedWindow("12 layer after NMS", CV_WINDOW_AUTOSIZE);
for (i = 0; i < counter; i++){
if (window[i].dropped == false){
cvRectangle(originImg1, cvPoint(window[i].x1, window[i].y1), cvPoint(window[i].x2, window[i].y2), cvScalar(255,0,0,0), 2, 4, 0);
sprintf(word, "%.2f", window[i].score);
cvPutText(originImg1, word, cvPoint(window[i].x1, window[i].y1), &font, cvScalar(255, 255, 255, 0));
}
}
cvShowImage("12 layer after NMS", originImg1);
cvMoveWindow("12 layer after NMS", 500, 10);
// 24 layer, 24 calibration, NMS
for (i = 0; i< counter; i++){
if (window[i].dropped == true) continue;
cvSetImageROI(srcImg, cvRect(window[i].x1, window[i].y1, window[i].x2 - window[i].x1, window[i].y2 - window[i].y1));
cvResize(srcImg, input24Img, CV_INTER_AREA);
data24 = (uchar*) input24Img->imageData;
preprocess(img24, data24, 0, 0, input24Img->widthStep, input24Img->nChannels, 24);
res_24Layer = Layer24(img24, 24, 24, input24Img->nChannels);
// 24 layer passed
if (res_24Layer > Threshold_24Layer){
// 24 calibration
out_24c = CaliLayer24(img24, 24, 24, input24Img->nChannels, Threshold_24CalibrationLayer);
s = out_24c[0];
x = out_24c[1];
y = out_24c[2];
free(out_24c);
cali_x = window[i].x1 - x * (window[i].x2 - window[i].x1) / s;
cali_y = window[i].y1 - y * (window[i].y2 - window[i].y1) / s;
cali_w = (window[i].x2 - window[i].x1) / s;
cali_h = (window[i].y2 - window[i].y1) / s;
// make sure the calibrated window not beyond the image boundary
if (cali_x >= WIDTH || cali_y >= HEIGHT) continue;
cali_x = max(cali_x, 0);
cali_y = max(cali_y, 0);
cali_w = min(cali_w, WIDTH - cali_x);
cali_h = min(cali_h, HEIGHT - cali_y);
window[i].x1 = cali_x; // x1
window[i].y1 = cali_y; // y1
window[i].x2 = cali_x + cali_w; // x2
window[i].y2 = cali_y + cali_h; // y2
window[i].score = res_24Layer; // 24 layer score
window[i].iou = 0.0; // iou ratio
window[i].dropped= false; // if it's dropped
}
else
{
window[i].dropped = true;
}
cvResetImageROI(srcImg);
}
printf("24 layer: x1: %d, y1: %d, x2: %d, y2: %d\n", window[15].x1, window[15].y1, window[15].x2, window[15].y2);
// NMS after 24 calibration
nms(window, counter, Threshold_24NMS);
// display sorted windows surviving 24 layer
cvNamedWindow("24 layer", CV_WINDOW_AUTOSIZE);
for (i = 0; i < counter; i++){
if (window[i].dropped == false){
cvRectangle(originImg2, cvPoint(window[i].x1, window[i].y1), cvPoint(window[i].x2, window[i].y2), cvScalar(255,0,0,0), 2, 4, 0);
sprintf(word, "%.2f", window[i].score);
cvPutText(originImg2, word, cvPoint(window[i].x1, window[i].y1), &font, cvScalar(255, 255, 255, 0));
}
}
cvShowImage("24 layer", originImg2);
cvMoveWindow("24 layer", 10, 400);
// end of 24 layer, 24 calibration, NMS
// 48 layer, 48 calibration, NMS
for (i = 0; i< counter; i++){
if (window[i].dropped == true) continue;
cvSetImageROI(srcImg, cvRect(window[i].x1, window[i].y1, window[i].x2 - window[i].x1, window[i].y2 - window[i].y1));
cvResize(srcImg, input48Img, CV_INTER_AREA);
data48 = (uchar*) input48Img->imageData;
preprocess(img48, data48, 0, 0, input48Img->widthStep, input48Img->nChannels, 48);
res_48Layer = Layer48(img48, 48, 48, input48Img->nChannels);
// 48 layer passed
if (res_48Layer > Threshold_48Layer){
// 48 calibration
out_48c = CaliLayer48(img48, 48, 48, input48Img->nChannels, Threshold_48CalibrationLayer);
s = out_48c[0];
x = out_48c[1];
y = out_48c[2];
free(out_48c);
cali_x = window[i].x1 - x * (window[i].x2 - window[i].x1) / s;
cali_y = window[i].y1 - y * (window[i].y2 - window[i].y1) / s;
cali_w = (window[i].x2 - window[i].x1) / s;
cali_h = (window[i].y2 - window[i].y1) / s;
// make sure the calibrated window not beyond the image boundary
if (cali_x >= WIDTH || cali_y >= HEIGHT) window[i].dropped = true;
cali_x = max(cali_x, 0);
cali_y = max(cali_y, 0);
cali_w = min(cali_w, WIDTH - cali_x);
cali_h = min(cali_h, HEIGHT - cali_y);
window[i].x1 = cali_x; // x1
window[i].y1 = cali_y; // y1
window[i].x2 = cali_x + cali_w; // x2
window[i].y2 = cali_y + cali_h; // y2
window[i].score = res_48Layer; // 48 layer score
window[i].iou = 0.0; // iou ratio
window[i].dropped= false; // if it's dropped
}
else
{
window[i].dropped = true;
}
cvResetImageROI(srcImg);
}
// NMS after 48 calibration
nms(window, counter, Threshold_48NMS);
// display sorted windows surviving 48 layer
cvNamedWindow("48 layer", CV_WINDOW_AUTOSIZE);
for (i = 0; i < counter; i++){
if (window[i].dropped == false){
cvRectangle(originImg3, cvPoint(window[i].x1, window[i].y1), cvPoint(window[i].x2, window[i].y2), cvScalar(255,0,0,0), 2, 4, 0);
sprintf(word, "%.2f", window[i].score);
cvPutText(originImg3, word, cvPoint(window[i].x1, window[i].y1), &font, cvScalar(255, 255, 255, 0));
}
}
cvShowImage("48 layer", originImg3);
cvMoveWindow("48 layer", 500, 400);
// end of 48 layer, 48 calibration, NMS
printf("48 layer: x1: %d, y1: %d, x2: %d, y2: %d, dropped: %s\n", window[15].x1, window[15].y1, window[15].x2, window[15].y2, window[15].dropped?"true":"false");
cvWaitKey(0);
cvDestroyWindow("12 layer");
cvDestroyWindow("12 layer after NMS");
cvDestroyWindow("24 layer");
cvDestroyWindow("48 layer");
pyr_rate *= 2;
if (dstImg->height / 2 < 12) flagStop = true;
}
// image pyramid loop ends
freeArray(img, 12);
freeArray(img24, 12);
freeArray(img48, 12);
return 0;
}
