double MaxEntModel::getObsCounts(EventSet& events, vector<double>& obsCounts)
{
vector<double> probs;
obsCounts.clear();
obsCounts.assign(_lambda.size(), 0);
double maxFtSum = 0;
for (unsigned int i = 0; i<events.size(); i++) {
MaxEntEvent& e = *events[i];
unsigned long c = e.classId();
double count = e.count();
double ftSum = 0;
for (unsigned long j=0; j<e.size(); j++) {
FtMap::iterator it = _index.find(e[j]);
if (it!=_index.end())
obsCounts[it->second+c] += count;
else { // new feature, need to expand obsCounts and _lambda
for (unsigned int k = 0; k<_classes; k++)
obsCounts.push_back(0);
obsCounts[_lambda.size()+c] += count;
addFeature(e[j]);
}
ftSum++;
}
if (ftSum>maxFtSum)
maxFtSum = ftSum;
}
return maxFtSum;
}
// In any order many at a time:
bool waitForEventSet(const EventSet& evts, Duration timeout=Duration::seconds(5)) {
EventSet read_evts;
for(uint idx = 0; idx < evts.size(); idx++) {
Event read_evt;
bool got_event = _events.blockingPop(read_evt, timeout);
TS_ASSERT(got_event);
if (!got_event) return false;
read_evts.insert(read_evt);
}
for(EventSet::iterator evt_it = evts.begin(), read_evt_it = read_evts.begin();
evt_it != evts.end(), read_evt_it != read_evts.end();
evt_it++, read_evt_it++)
{
TS_ASSERT_EQUALS(*evt_it, *read_evt_it);
if (*evt_it != *read_evt_it) return false;
}
return true;
}
void MaxEntTrainer::Add_Event(EventSet &events, const char *name, const char *data)
{
const string delims(" ");
MaxEntEvent *event = new MaxEntEvent;
getIds(data, *event, delims);
event->classId(getClassId(name));
event->count(1);
events.push_back(event);
}
void MaxEntTrainer::readEvents(istream& istrm, EventSet& events)
{
string line, cls;
const string delims(" ");
while ((istrm>>cls) && getline(istrm, line)) {
MaxEntEvent* event = new MaxEntEvent;
getIds(line, *event, delims);
event->classId(getClassId(cls));
event->count(1);
events.push_back(event);
}
}
static void UpdateEventMap()
{
EventMap map;
Thread::Mutex::Scoped l(locker_event);
for(IOMap::const_iterator it = ionew.begin(), ie = ionew.end(); it != ie; ++it)
{
PollIO *io = (*it).second;
iomap[(*it).first] = io;
if (io->newevent)
eventset.insert(io);
}
ionew.clear();
for (EventSet::iterator it = eventset.begin(), ie = eventset.end(); it != ie; ++it)
{
map.insert( std::make_pair(*it, (*it)->event) );
(*it)->update = false;
(*it)->event = (*it)->newevent;
(*it)->newevent = 0;
}
eventset.clear();
wakeup_flag = true;
std::for_each(map.begin(), map.end(), std::ptr_fun(&LoadEvent));
}
double MaxEntTrainer::test(EventSet& events, MaxEntModel& model)
{
vector<double> probs;
double total=0, error = 0;
for (unsigned int i = 0; i<events.size(); i++) {
int c = model.getProbs(*events[i], probs);
if (c!=(int)events[i]->classId()) {
error++;
if (_printDetails) cerr << '*';
}
if (_printDetails) {
cerr << className(events[i]->classId()) << '\t';
for (unsigned int cl = 0; cl<probs.size(); cl++) {
cerr << className(cl) << ' ' << probs[cl] << '\t';
}
cerr << endl;
}
total++;
}
return error/total;
}
double MaxEntModel::getExpects(EventSet& events, vector<double>& expects)
{
expects.clear();
expects.assign(_lambda.size(), 0);
double sumLogProb = 0;
for (unsigned int i = 0; i<events.size(); i++) {
MaxEntEvent& e = *events[i];
vector<double> probs;
getProbs(e, probs);
for (unsigned long c = 0; c < _classes; ++c) {
double count = probs[c] * e.count();
for (unsigned long j = 0; j < e.size(); j++) {
FtMap::iterator it = _index.find(e[j]);
if (it!=_index.end())
expects[it->second + c] += count;
}
}
sumLogProb += log(probs[e.classId()]);
}
return sumLogProb;
}