Float evaluate_sequences(vector<vector<int> > &goldS,vector<vector<int> > &predicted,vector<string> &LIS ,int num_labels,vector<verbose_res> &VR){
int numtag,numctag,numbtag;
numtag=numctag=numbtag=0;
int num_class = ( num_labels / 2 ) + 1;
vector<int> ccount(num_class,0),pcount(num_class,0),ncorrect(num_class,0),
nleft(num_class,0),nright(num_class,0),
nbcorrect(num_class,0),nbleft(num_class,0),nbright(num_class,0);
int match,bmatch;
match=bmatch=0;
int num_seq = goldS.size();
for (int i = 0; i < num_seq ; ++i){ // For each sequence
unsigned int N = goldS[i].size();
if ( N != predicted[i].size() ) {
cerr << "Unmatching length of labels for sentence" << i << "\n"; abort();
}
for ( unsigned int n = 0 ; n < N ; ++n, ++numtag ) {
int clabel = goldS[i][n];
int cclass = class_type( clabel );
int cbraket = braket_type( n, goldS[i] );
if ( cbraket == SBrac || cbraket == BBrac ) {
ccount[cclass]++;
}
int plabel = predicted[i][n];
int pclass = class_type( plabel );
int pbraket = braket_type( n, predicted[i] );
if ( pbraket == SBrac || pbraket == BBrac ) {
pcount[pclass]++;
}
if ( cbraket == pbraket ) numbtag++;
if ( clabel == plabel && cbraket == pbraket ) numctag++;
if ( pbraket == SBrac && cbraket == SBrac ) {
nbcorrect[pclass]++; nbleft[pclass]++; nbright[pclass]++;
if( pclass == cclass ) {
ncorrect[pclass]++; nleft[pclass]++; nright[pclass]++;
}
}
else if ( pbraket == SBrac && cbraket == EBrac ) {
nbright[pclass]++;
if( pclass == cclass ) { nright[pclass]++; }
}
else if ( pbraket == SBrac && cbraket == BBrac ) {
nbleft[pclass]++;
if( pclass == cclass ) { nleft[pclass]++; }
}
else if ( pbraket == BBrac && cbraket == SBrac ) {
nbleft[pclass]++;
if( pclass == cclass ) { nleft[pclass]++; }
}
else if ( pbraket == BBrac && cbraket == BBrac ) {
nbleft[pclass]++; bmatch=1;
if( pclass == cclass ) { nleft[pclass]++; match=1;}
}
if ( pbraket != cbraket || pbraket == none ) bmatch = 0;
if ( cclass != pclass || !bmatch ) match = 0;
if ( pbraket == EBrac && cbraket == SBrac ) {
nbright[pclass]++;
if( pclass == cclass ) { nright[pclass]++; }
}
if ( pbraket == EBrac && cbraket == EBrac ) {
if( bmatch ) nbcorrect[pclass]++;
nbright[pclass]++;
if( pclass == cclass ) {
if ( match && ncorrect[pclass] <= pcount[pclass])
ncorrect[pclass]++;
nright[pclass]++;
}
}
}
}
Float score;
int numans = getsum(pcount), numref = getsum(ccount);
verbose_res vr("ALL",int(numref),int(numans));
if ( !numref ) cerr << "\n\nNo names in reference!\n";
if ( !numans ) cerr << "\n\nNo names in answers!\n";
Float ret_val = compute_score(getsum(ncorrect),numans,numref,"FULLY CORRECT answer",vr);
VR.push_back(vr);
if (num_class > 2)
for( unsigned int cl = 1 ; cl < pcount.size() ; ++cl ) {
string cl_str = LIS[reverse_class_type(cl)];
verbose_res vr_cl(string(cl_str,2,int(cl_str.size()-2)),ccount[cl],pcount[cl]);
score = compute_score(ncorrect[cl],pcount[cl],ccount[cl],"FULLY CORRECT answer",vr_cl);
VR.push_back(vr_cl);
}
if (ret_val>1.0) ret_val=1.0;
return ret_val;
}
Float evaluate_sequences(vector<vector<int> > &goldS,
vector<vector<int> > &predicted,
vector<string> &LIS ,
int num_labels,
int verbose,
ofstream &out){
int numtag,numctag,numbtag;
numtag=numctag=numbtag=0;
int num_class = ( num_labels / 2 ) + 1;
vector<int> ccount(num_class,0),pcount(num_class,0),ncorrect(num_class,0),
nleft(num_class,0),nright(num_class,0),
nbcorrect(num_class,0),nbleft(num_class,0),nbright(num_class,0);
int match,bmatch;
match=bmatch=0;
int num_seq = goldS.size();
for (int i = 0; i < num_seq ; ++i){
// For each sequence
unsigned int N = goldS[i].size();
if ( N != predicted[i].size() ) {
cerr << "Unmatching length of labels for sentence" << i << "\n";
abort();
}
for ( unsigned int n = 0 ; n < N ; ++n, ++numtag ) {
int clabel = goldS[i][n];
int cclass = class_type( clabel );
int cbraket = braket_type( n, goldS[i] );
if ( cbraket == SBrac || cbraket == BBrac ) {
ccount[cclass]++;
}
int plabel = predicted[i][n];
int pclass = class_type( plabel );
int pbraket = braket_type( n, predicted[i] );
if ( pbraket == SBrac || pbraket == BBrac ) {
pcount[pclass]++;
}
if ( cbraket == pbraket ) numbtag++;
if ( clabel == plabel && cbraket == pbraket ) numctag++;
if ( pbraket == SBrac && cbraket == SBrac ) {
nbcorrect[pclass]++; nbleft[pclass]++; nbright[pclass]++;
if( pclass == cclass ) {
ncorrect[pclass]++; nleft[pclass]++; nright[pclass]++;
}
}
else if ( pbraket == SBrac && cbraket == EBrac ) {
nbright[pclass]++;
if( pclass == cclass ) { nright[pclass]++; }
}
else if ( pbraket == SBrac && cbraket == BBrac ) {
nbleft[pclass]++;
if( pclass == cclass ) { nleft[pclass]++; }
}
else if ( pbraket == BBrac && cbraket == SBrac ) {
nbleft[pclass]++;
if( pclass == cclass ) { nleft[pclass]++; }
}
else if ( pbraket == BBrac && cbraket == BBrac ) {
nbleft[pclass]++; bmatch=1;
if( pclass == cclass ) { nleft[pclass]++; match=1;}
}
/*
if ( pbraket != cbraket ) bmatch = 0;
if (!(clabel == plabel && cbraket == pbraket ) ) match = 0;
if ( pbraket == none || cbraket == none ) match=bmatch=0;
*/
if ( pbraket != cbraket || pbraket == none ) bmatch = 0;
if ( cclass != pclass || !bmatch ) match = 0;
if ( pbraket == EBrac && cbraket == SBrac ) {
nbright[pclass]++;
if( pclass == cclass ) { nright[pclass]++; }
}
if ( pbraket == EBrac && cbraket == EBrac ) {
if( bmatch ) nbcorrect[pclass]++;
nbright[pclass]++;
if( pclass == cclass ) {
if ( match && ncorrect[pclass] <= pcount[pclass])
ncorrect[pclass]++;
nright[pclass]++;
}
}
}
}
Float score;
int numans = getsum( pcount );
int numref = getsum( ccount );
if (verbose) {
out << "\n[Tagging Performance]\n";
out << "# of tags: " << numtag << ",\t correct tags: " << numctag
<< ",\t correct IOBs: " << numbtag << "\n";
out << "precision with class info: " << (float)numctag/numtag;
out << ",\t w/o class info: " << (float)numbtag/numtag << "\n";
out << "\n[Object Identification Performance]\n";
out << "# of OBJECTs: " << numref << "\t ANSWERs: " << numans << ".\n";
out << "\n# (recall / precision / f-score) of ...\n";
}
if ( !numref ) cerr << "\n\nNo names in reference!\n";
if ( !numans ) cerr << "\n\nNo names in answers!\n";
Float ret_val = compute_score( getsum(ncorrect), numans, numref ,
"FULLY CORRECT answer", verbose, out);
/* There is a bug on the right
score = compute_score( getsum(nleft), numans, numref ,
"correct LEFT boundary", verbose, out );
score = compute_score( getsum(nright), numans, numref ,
"correct RIGHT boundary", verbose, out );
*/
if (num_class > 2 && verbose )
for( unsigned int cl = 1 ; cl < pcount.size() ; ++cl ) {
string cl_str = LIS[reverse_class_type(cl)];
out << "\n[" << cl << "/" << string(cl_str,2,int(cl_str.size()-2)) << "\tIdentification Performance]\n";
out << "# of OBJECTs: " << ccount[cl] << ",\t ANSWERs: " << pcount[cl];
out << "\n# (recall / precision / f-score) of ...\n";
score = compute_score( ncorrect[cl], pcount[cl], ccount[cl] ,
"FULLY CORRECT answer", verbose, out );
/*
score = compute_score( nleft[cl], pcount[cl], ccount[cl] ,
"correct LEFT boundary", verbose, out );
score = compute_score( nright[cl], pcount[cl], ccount[cl] ,
"correct RIGHT boundary", verbose, out );
*/
}
else if ( !verbose )
for( unsigned int cl = 1 ; cl < pcount.size() ; ++cl ) {
score = compute_score( ncorrect[cl], pcount[cl], ccount[cl] ,
"FULLY CORRECT answer", verbose, out );
cerr << "Cl:" << cl << " " << score << " ";
}
//assert(ret_val<=1.0);
if (ret_val>1.0) ret_val=1.0;
return ret_val;
}
int lowess(const ContainerType& x,
const ContainerType& y,
double frac, // parameter f
int nsteps,
ValueType delta,
ContainerType& ys,
ContainerType& resid_weights, // vector rw
ContainerType& weights // vector res
)
{
bool fit_ok;
size_t ns, n(x.size());
if (n < 2)
{
ys[0] = y[0];
return 1;
}
// how many points around estimation point should be used for regression:
// at least two, at most n points
size_t tmp = (size_t)(frac * (double)n);
ns = std::max(std::min(tmp, n), (size_t)2);
// robustness iterations
for (int iter = 1; iter <= nsteps + 1; iter++)
{
// start of array in C++ at 0 / in FORTRAN at 1
// last: index of prev estimated point
// i: index of current point
size_t i(0), last(-1), nleft(0), nright(ns -1);
// Fit all data points y[i] until the end of the array
do
{
// Identify the neighborhood around the current x[i]
// -> get the nearest ns points
update_neighborhood(x, n, i, nleft, nright);
// Calculate weights and apply fit (original lowest function)
fit_ok = lowest(x, y, n, x[i], ys[i], nleft, nright,
weights, (iter > 1), resid_weights);
// if something went wrong during the fit, use y[i] as the
// fitted value at x[i]
if (!fit_ok) ys[i] = y[i];
// If we skipped some points (because of how delta was set), go back
// and fit them by linear interpolation.
if (last < i - 1)
{
interpolate_skipped_fits(x, i, last, ys);
}
// Update the last fit counter to indicate we've now fit this point.
// Find the next i for which we'll run a regression.
update_indices(x, n, delta, i, last, ys);
}
while (last < n - 1);
// compute current residuals
for (i = 0; i < n; i++)
{
weights[i] = y[i] - ys[i];
}
// compute robustness weights except last time
if (iter > nsteps) break;
calculate_residual_weights(n, weights, resid_weights);
}
return 0;
}