void OutputFeatureScores( std::ostream& out
, const ScoreComponentCollection &features
, const FeatureFunction *ff
, std::string &lastName )
{
const StaticData &staticData = StaticData::Instance();
bool labeledOutput = staticData.IsLabeledNBestList();
// regular features (not sparse)
if (ff->GetNumScoreComponents() != 0) {
if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) {
lastName = ff->GetScoreProducerDescription();
out << " " << lastName << "=";
}
vector<float> scores = features.GetScoresForProducer( ff );
for (size_t j = 0; j<scores.size(); ++j) {
out << " " << scores[j];
}
}
// sparse features
const FVector scores = features.GetVectorForProducer( ff );
for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) {
out << " " << i->first << "= " << i->second;
}
}
void FVector::updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts)
{
for (size_t i = 0; i < weightUpdate.m_coreFeatures.size(); ++i) {
if (signedCounts) {
//int sign = weightUpdate.m_coreFeatures[i] >= 0 ? 1 : -1;
//m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]) * sign;
m_coreFeatures[i] += weightUpdate.m_coreFeatures[i];
} else
//m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]);
m_coreFeatures[i] += abs(weightUpdate.m_coreFeatures[i]);
}
for (const_iterator i = weightUpdate.cbegin(); i != weightUpdate.cend(); ++i) {
if (weightUpdate[i->first] == 0)
continue;
float value = get(i->first);
if (signedCounts) {
//int sign = weightUpdate[i->first] >= 0 ? 1 : -1;
//value += (weightUpdate[i->first] * weightUpdate[i->first]) * sign;
value += weightUpdate[i->first];
} else
//value += (weightUpdate[i->first] * weightUpdate[i->first]);
value += abs(weightUpdate[i->first]);
set(i->first, value);
}
}
void IOWrapper::OutputSparseFeatureScores( std::ostream& out, const ChartTrellisPath &path, const FeatureFunction *ff, std::string &lastName )
{
const StaticData &staticData = StaticData::Instance();
bool labeledOutput = staticData.IsLabeledNBestList();
const FVector scores = path.GetScoreBreakdown().GetVectorForProducer( ff );
// report weighted aggregate
if (! ff->GetSparseFeatureReporting()) {
const FVector &weights = staticData.GetAllWeights().GetScoresVector();
if (labeledOutput && !boost::contains(ff->GetScoreProducerDescription(), ":"))
out << " " << ff->GetScoreProducerWeightShortName() << ":";
out << " " << scores.inner_product(weights);
}
// report each feature
else {
for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) {
if (i->second != 0) { // do not report zero-valued features
if (labeledOutput)
out << " " << i->first << ":";
out << " " << i->second;
}
}
}
}
// count non-zero occurrences for all sparse features
void FVector::setToBinaryOf(const FVector& rhs)
{
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i)
if (rhs.get(i->first) != 0)
set(i->first, 1);
for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i)
m_coreFeatures[i] = 1;
}
void FVector::updateLearningRates(float decay_core, float decay_sparse, const FVector &confidenceCounts, float core_r0, float sparse_r0)
{
for (size_t i = 0; i < confidenceCounts.m_coreFeatures.size(); ++i) {
m_coreFeatures[i] = 1.0/(1.0/core_r0 + decay_core * abs(confidenceCounts.m_coreFeatures[i]));
}
for (const_iterator i = confidenceCounts.cbegin(); i != confidenceCounts.cend(); ++i) {
float value = 1.0/(1.0/sparse_r0 + decay_sparse * abs(i->second));
set(i->first, value);
}
}
bool FVector::operator== (const FVector& rhs) const
{
if (this == &rhs) {
return true;
}
if (m_coreFeatures.size() != rhs.m_coreFeatures.size()) {
return false;
}
for (size_t i = 0; i < m_coreFeatures.size(); ++i) {
if (!equalsTolerance(m_coreFeatures[i], rhs.m_coreFeatures[i])) return false;
}
for (const_iterator i = cbegin(); i != cend(); ++i) {
if (!equalsTolerance(i->second,rhs.get(i->first))) return false;
}
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) {
if (!equalsTolerance(i->second, get(i->first))) return false;
}
return true;
}
void OutputFeatureScores( std::ostream& out, const TrellisPath &path, const FeatureFunction *ff, std::string &lastName )
{
const StaticData &staticData = StaticData::Instance();
bool labeledOutput = staticData.IsLabeledNBestList();
// regular features (not sparse)
if (ff->GetNumScoreComponents() != ScoreProducer::unlimited) {
if( labeledOutput && lastName != ff->GetScoreProducerWeightShortName() ) {
lastName = ff->GetScoreProducerWeightShortName();
out << " " << lastName << ":";
}
vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( ff );
for (size_t j = 0; j<scores.size(); ++j) {
out << " " << scores[j];
}
}
// sparse features
else {
const FVector scores = path.GetScoreBreakdown().GetVectorForProducer( ff );
// report weighted aggregate
if (! ff->GetSparseFeatureReporting()) {
const FVector &weights = staticData.GetAllWeights().GetScoresVector();
if (labeledOutput && !boost::contains(ff->GetScoreProducerDescription(), ":"))
out << " " << ff->GetScoreProducerWeightShortName() << ":";
out << " " << scores.inner_product(weights);
}
// report each feature
else {
for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++)
out << " " << i->first << ": " << i->second;
/* if (i->second != 0) { // do not report zero-valued features
float weight = staticData.GetSparseWeight(i->first);
if (weight != 0)
out << " " << i->first << "=" << weight;
}*/
}
}
}
// add only sparse features
void FVector::sparsePlusEquals(const FVector& rhs)
{
for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i)
set(i->first, get(i->first) + i->second);
}
