Float evaluate_sequences(vector<vector<int> > &goldS, vector<vector<int> > &predicted, vector<string> &LIS , int num_labels, int verbose, ofstream &out){ int numtag,numctag,numbtag; numtag=numctag=numbtag=0; int num_class = ( num_labels / 2 ) + 1; vector<int> ccount(num_class,0),pcount(num_class,0),ncorrect(num_class,0), nleft(num_class,0),nright(num_class,0), nbcorrect(num_class,0),nbleft(num_class,0),nbright(num_class,0); int match,bmatch; match=bmatch=0; int num_seq = goldS.size(); for (int i = 0; i < num_seq ; ++i){ // For each sequence unsigned int N = goldS[i].size(); if ( N != predicted[i].size() ) { cerr << "Unmatching length of labels for sentence" << i << "\n"; abort(); } for ( unsigned int n = 0 ; n < N ; ++n, ++numtag ) { int clabel = goldS[i][n]; int cclass = class_type( clabel ); int cbraket = braket_type( n, goldS[i] ); if ( cbraket == SBrac || cbraket == BBrac ) { ccount[cclass]++; } int plabel = predicted[i][n]; int pclass = class_type( plabel ); int pbraket = braket_type( n, predicted[i] ); if ( pbraket == SBrac || pbraket == BBrac ) { pcount[pclass]++; } if ( cbraket == pbraket ) numbtag++; if ( clabel == plabel && cbraket == pbraket ) numctag++; if ( pbraket == SBrac && cbraket == SBrac ) { nbcorrect[pclass]++; nbleft[pclass]++; nbright[pclass]++; if( pclass == cclass ) { ncorrect[pclass]++; nleft[pclass]++; nright[pclass]++; } } else if ( pbraket == SBrac && cbraket == EBrac ) { nbright[pclass]++; if( pclass == cclass ) { nright[pclass]++; } } else if ( pbraket == SBrac && cbraket == BBrac ) { nbleft[pclass]++; if( pclass == cclass ) { nleft[pclass]++; } } else if ( pbraket == BBrac && cbraket == SBrac ) { nbleft[pclass]++; if( pclass == cclass ) { nleft[pclass]++; } } else if ( pbraket == BBrac && cbraket == BBrac ) { nbleft[pclass]++; bmatch=1; if( pclass == cclass ) { nleft[pclass]++; match=1;} } /* if ( pbraket != cbraket ) bmatch = 0; if (!(clabel == plabel && cbraket == pbraket ) ) match = 0; if ( pbraket == none || cbraket == none ) match=bmatch=0; */ if ( pbraket != cbraket || pbraket == none ) bmatch = 0; if ( cclass != pclass || !bmatch ) match = 0; if ( pbraket == EBrac && cbraket == SBrac ) { nbright[pclass]++; if( pclass == cclass ) { nright[pclass]++; } } if ( pbraket == EBrac && cbraket == EBrac ) { if( bmatch ) nbcorrect[pclass]++; nbright[pclass]++; if( pclass == cclass ) { if ( match && ncorrect[pclass] <= pcount[pclass]) ncorrect[pclass]++; nright[pclass]++; } } } } Float score; int numans = getsum( pcount ); int numref = getsum( ccount ); if (verbose) { out << "\n[Tagging Performance]\n"; out << "# of tags: " << numtag << ",\t correct tags: " << numctag << ",\t correct IOBs: " << numbtag << "\n"; out << "precision with class info: " << (float)numctag/numtag; out << ",\t w/o class info: " << (float)numbtag/numtag << "\n"; out << "\n[Object Identification Performance]\n"; out << "# of OBJECTs: " << numref << "\t ANSWERs: " << numans << ".\n"; out << "\n# (recall / precision / f-score) of ...\n"; } if ( !numref ) cerr << "\n\nNo names in reference!\n"; if ( !numans ) cerr << "\n\nNo names in answers!\n"; Float ret_val = compute_score( getsum(ncorrect), numans, numref , "FULLY CORRECT answer", verbose, out); /* There is a bug on the right score = compute_score( getsum(nleft), numans, numref , "correct LEFT boundary", verbose, out ); score = compute_score( getsum(nright), numans, numref , "correct RIGHT boundary", verbose, out ); */ if (num_class > 2 && verbose ) for( unsigned int cl = 1 ; cl < pcount.size() ; ++cl ) { string cl_str = LIS[reverse_class_type(cl)]; out << "\n[" << cl << "/" << string(cl_str,2,int(cl_str.size()-2)) << "\tIdentification Performance]\n"; out << "# of OBJECTs: " << ccount[cl] << ",\t ANSWERs: " << pcount[cl]; out << "\n# (recall / precision / f-score) of ...\n"; score = compute_score( ncorrect[cl], pcount[cl], ccount[cl] , "FULLY CORRECT answer", verbose, out ); /* score = compute_score( nleft[cl], pcount[cl], ccount[cl] , "correct LEFT boundary", verbose, out ); score = compute_score( nright[cl], pcount[cl], ccount[cl] , "correct RIGHT boundary", verbose, out ); */ } else if ( !verbose ) for( unsigned int cl = 1 ; cl < pcount.size() ; ++cl ) { score = compute_score( ncorrect[cl], pcount[cl], ccount[cl] , "FULLY CORRECT answer", verbose, out ); cerr << "Cl:" << cl << " " << score << " "; } //assert(ret_val<=1.0); if (ret_val>1.0) ret_val=1.0; return ret_val; }
Float evaluate_sequences(vector<vector<int> > &goldS,vector<vector<int> > &predicted,vector<string> &LIS ,int num_labels,vector<verbose_res> &VR){ int numtag,numctag,numbtag; numtag=numctag=numbtag=0; int num_class = ( num_labels / 2 ) + 1; vector<int> ccount(num_class,0),pcount(num_class,0),ncorrect(num_class,0), nleft(num_class,0),nright(num_class,0), nbcorrect(num_class,0),nbleft(num_class,0),nbright(num_class,0); int match,bmatch; match=bmatch=0; int num_seq = goldS.size(); for (int i = 0; i < num_seq ; ++i){ // For each sequence unsigned int N = goldS[i].size(); if ( N != predicted[i].size() ) { cerr << "Unmatching length of labels for sentence" << i << "\n"; abort(); } for ( unsigned int n = 0 ; n < N ; ++n, ++numtag ) { int clabel = goldS[i][n]; int cclass = class_type( clabel ); int cbraket = braket_type( n, goldS[i] ); if ( cbraket == SBrac || cbraket == BBrac ) { ccount[cclass]++; } int plabel = predicted[i][n]; int pclass = class_type( plabel ); int pbraket = braket_type( n, predicted[i] ); if ( pbraket == SBrac || pbraket == BBrac ) { pcount[pclass]++; } if ( cbraket == pbraket ) numbtag++; if ( clabel == plabel && cbraket == pbraket ) numctag++; if ( pbraket == SBrac && cbraket == SBrac ) { nbcorrect[pclass]++; nbleft[pclass]++; nbright[pclass]++; if( pclass == cclass ) { ncorrect[pclass]++; nleft[pclass]++; nright[pclass]++; } } else if ( pbraket == SBrac && cbraket == EBrac ) { nbright[pclass]++; if( pclass == cclass ) { nright[pclass]++; } } else if ( pbraket == SBrac && cbraket == BBrac ) { nbleft[pclass]++; if( pclass == cclass ) { nleft[pclass]++; } } else if ( pbraket == BBrac && cbraket == SBrac ) { nbleft[pclass]++; if( pclass == cclass ) { nleft[pclass]++; } } else if ( pbraket == BBrac && cbraket == BBrac ) { nbleft[pclass]++; bmatch=1; if( pclass == cclass ) { nleft[pclass]++; match=1;} } if ( pbraket != cbraket || pbraket == none ) bmatch = 0; if ( cclass != pclass || !bmatch ) match = 0; if ( pbraket == EBrac && cbraket == SBrac ) { nbright[pclass]++; if( pclass == cclass ) { nright[pclass]++; } } if ( pbraket == EBrac && cbraket == EBrac ) { if( bmatch ) nbcorrect[pclass]++; nbright[pclass]++; if( pclass == cclass ) { if ( match && ncorrect[pclass] <= pcount[pclass]) ncorrect[pclass]++; nright[pclass]++; } } } } Float score; int numans = getsum(pcount), numref = getsum(ccount); verbose_res vr("ALL",int(numref),int(numans)); if ( !numref ) cerr << "\n\nNo names in reference!\n"; if ( !numans ) cerr << "\n\nNo names in answers!\n"; Float ret_val = compute_score(getsum(ncorrect),numans,numref,"FULLY CORRECT answer",vr); VR.push_back(vr); if (num_class > 2) for( unsigned int cl = 1 ; cl < pcount.size() ; ++cl ) { string cl_str = LIS[reverse_class_type(cl)]; verbose_res vr_cl(string(cl_str,2,int(cl_str.size()-2)),ccount[cl],pcount[cl]); score = compute_score(ncorrect[cl],pcount[cl],ccount[cl],"FULLY CORRECT answer",vr_cl); VR.push_back(vr_cl); } if (ret_val>1.0) ret_val=1.0; return ret_val; }
inline ss_sint_t sinteger64::compare(long i) const { return compare(class_type(0, i)); }
inline sinteger64::class_type& sinteger64::operator %=(long i) { return operator %=(class_type(i)); }
inline sinteger64::class_type& sinteger64::operator %=(ss_sint32_t i) { return operator %=(class_type(i)); }
inline sinteger64::class_type& sinteger64::operator /=(int i) { return operator /=(class_type(i)); }