void process(char* filename, mapT &map)
{
int nbWords=0;
int nbUniqueWords=0;
cout<<filename<<endl;
fstream f;
f.open(filename, ios::in | ios::binary);
if (f.bad())
{
f.close();
throw "Could not open file";
}
f.unsetf(ios_base::skipws);
char buffer[128];
int pos=0;
char c;
int state=0;
mapT::iterator it;
while (!f.eof())
{
f>>c;
if ( (c>='a' && c<='z') || (c>='A' && c<='Z') || (c>='0' && c<='9') || c=='ç' || c=='ô' ||
c=='è' || c=='ê' || c=='é' || c=='û' || c=='ù' || c=='à' || c=='â' || c=='î' || c=='ï')
{
state=1;
buffer[pos]=c;
pos++;
} else
{
if (pos>0)
{
buffer[pos]=0;
char* tmp = new char[pos+1];
memcpy(tmp, buffer, pos+1);
it=map.find(tmp);
if (it==map.end())
{ // new element
map[tmp]=0;
nbUniqueWords++;
} else
{
map[tmp]++;
}
//cout<<buffer<<endl;
pos=0;
nbWords++;
}
state=0;
}
};
f.close();
cout<<nbUniqueWords <<" unique words out of " <<nbWords<<" words."<<endl;
}
void
LoadFilterMap(mapT& filterMap, char* filterFile)
{
boost::filesystem::ifstream inFile(filterFile);
filterMap.empty();
while (inFile)
{
std::string s;
std::getline( inFile, s );
if (inFile)
filterMap[s] = 1;
}
}
static void serialize(archiveT & ar, mapT & map, const unsigned int /*version*/)
{
// force 64 bit for portability
boost::int64_t size = 0;
ar.load_binary(&size , sizeof(size));
map.resize(boost::numeric_cast<size_t>(size)); // make sure the cast back to size_t is safe
typename mapT::key_type k;
while(size-- > 0)
{
ar >> k;
ar >> map[k];
}
}
/* Generate 2000 or less characters based on the seedMap passed as the argument
* in accordance with the Markov model
*/
void generateRandomText(mapT &seedMap) {
string seed = mostFrequentSeed(seedMap);
string result;
for (int i = 0; i < NUM_CHAR; i++) {
if (seedMap.containsKey(seed)) {
int low = 0;
int high = seedMap[seed].size() - 1;
int randInt = randomInteger(low, high);
char nextCh = seedMap[seed].get(randInt);
result += nextCh;
seed.erase(0, 1);
seed += nextCh;
}
}
cout << result << endl;
}
iterator insert (iterator position, value_type val)
{
wxMutexLocker lock(m_mutex);
return map.insert( position, val);
}
iterator end()
{
wxMutexLocker lock(m_mutex);
return map.end();
}
iterator begin()
{
wxMutexLocker lock(m_mutex);
return map.begin();
}
iterator find (const key_type& k)
{
wxMutexLocker lock(m_mutex);
return map.find(k);
}
size_type erase (const key_type& k)
{
wxMutexLocker lock(m_mutex);
return map.erase(k);
}
void erase (iterator position)
{
wxMutexLocker lock(m_mutex);
map.erase(position);
}