/* Look for mime handler in pool */
static RecollFilter *getMimeHandlerFromCache(const string& key)
{
PTMutexLocker locker(o_handlers_mutex);
string xdigest;
MD5HexPrint(key, xdigest);
LOGDEB(("getMimeHandlerFromCache: %s cache size %u\n",
xdigest.c_str(), o_handlers.size()));
multimap<string, RecollFilter *>::iterator it = o_handlers.find(key);
if (it != o_handlers.end()) {
RecollFilter *h = it->second;
hlruit_tp it1 = find(o_hlru.begin(), o_hlru.end(), it);
if (it1 != o_hlru.end()) {
o_hlru.erase(it1);
} else {
LOGERR(("getMimeHandlerFromCache: lru position not found\n"));
}
o_handlers.erase(it);
LOGDEB(("getMimeHandlerFromCache: %s found size %u\n",
xdigest.c_str(), o_handlers.size()));
return h;
}
LOGDEB(("getMimeHandlerFromCache: %s not found\n", xdigest.c_str()));
return 0;
}
void CDetCandit::Erasing_Prescribed_Candidates(multimap<double, size_t>& FeedbackCase_mm)
{
if((int)FeedbackCase_mm.size() > FEEDBACK_CASES_NUM){
for(int num = FeedbackCase_mm.size() - FEEDBACK_CASES_NUM; num > 0; num--){
FeedbackCase_mm.erase(FeedbackCase_mm.begin());
}
}
}
void operator& (multimap<double,double> x, ostream& stream) {
x.size() & stream;
for (multimap<double,double>::const_iterator pos = x.begin (); pos != x.end() ; ++pos) {
pos->first & stream;
pos->second & stream;
}
}
int Slave::checkBadDest(multimap<int64_t, Address>& sndspd, vector<Address>& bad)
{
bad.clear();
if (sndspd.empty())
return 0;
int m = sndspd.size() / 2;
multimap<int64_t, Address>::iterator p = sndspd.begin();
for (int i = 0; i < m; ++ i)
++ p;
int64_t median = p->first;
int locpos = 0;
for (multimap<int64_t, Address>::iterator i = sndspd.begin(); i != sndspd.end(); ++ i)
{
if (i->first > (median / 2))
return bad.size();
bad.push_back(i->second);
locpos ++;
}
return bad.size();
}
void Vocabulary::search_image(int imgNum, vector<int> images, multimap<int, int> &imageScore, vector<int> &rankedScore)
{
for(int i=0; i<imgNum; i++)
{
imageScore.insert(pair<int,int>(i+1,0));
}
cout<<imageScore.size()<<endl;
int score=0;
for(auto ite=imageScore.begin(); ite!=imageScore.end(); ite++)
{
for(int i=0; i<images.size(); i++)
{
if((*ite).first==images[i])
{
(*ite).second++;
}
}
///normalize
float normalized_score=(float)(*ite).second/(float)images.size();
// (*ite).second=normalized_score;
cout.precision(5);
cout<<(*ite).first<<" "<<(*ite).second<<" "<<normalized_score<<endl;
rankedScore.push_back((*ite).second);
}
}
bool CMesh::RayCast(VECTOR4D & RayOrigin, VECTOR4D & RayDir, multimap<float, unsigned long>& Vertices, float radius) {
for (size_t i = 0; i < m_Vertices.size(); i++) {
if (RaySphereIntersect(RayOrigin, RayDir, m_Vertices[i].Position, radius)) {
float dist = Magnity(m_Vertices[i].Position - RayOrigin);
Vertices.insert(make_pair(dist, (unsigned long)i));
}
}
return Vertices.size() > 0;
}
/* Return mime handler to pool */
void returnMimeHandler(RecollFilter *handler)
{
typedef multimap<string, RecollFilter*>::value_type value_type;
if (handler == 0) {
LOGERR(("returnMimeHandler: bad parameter\n"));
return;
}
handler->clear();
PTMutexLocker locker(o_handlers_mutex);
LOGDEB(("returnMimeHandler: returning filter for %s cache size %d\n",
handler->get_mime_type().c_str(), o_handlers.size()));
// Limit pool size. The pool can grow quite big because there are
// many filter types, each of which can be used in several copies
// at the same time either because it occurs several times in a
// stack (ie mail attachment to mail), or because several threads
// are processing the same mime type at the same time.
multimap<string, RecollFilter *>::iterator it;
if (o_handlers.size() >= max_handlers_cache_size) {
static int once = 1;
if (once) {
once = 0;
for (it = o_handlers.begin(); it != o_handlers.end(); it++) {
LOGDEB1(("Cache full. key: %s\n", it->first.c_str()));
}
LOGDEB1(("Cache LRU size: %u\n", o_hlru.size()));
}
if (o_hlru.size() > 0) {
it = o_hlru.back();
o_hlru.pop_back();
delete it->second;
o_handlers.erase(it);
}
}
it = o_handlers.insert(value_type(handler->get_id(), handler));
o_hlru.push_front(it);
}
bool CMesh::RayCast(VECTOR4D &RayOrigin, VECTOR4D &RayDir, multimap<float, unsigned long> &Vertices, float radius)
{
// Ecuacion de la esfera con centro en el origen
// x^2 + y^2 + z^2 = r^2
for (unsigned long index = 0; index < m_Vertexes.size(); ++index)
{
if (RaySphereIntersect(RayOrigin, RayDir, m_Vertexes[index].Position, radius))
{
Vertices.insert(make_pair(Magnity(m_Vertexes[index].Position - RayOrigin), index));
}
}
return Vertices.size() != 0;
}
list < Attivita > ShortestJobFirst ( multimap < int, Attivita > &Activities ){
list < Attivita > A;
multimap< int, Attivita >::iterator it_ActivitiesMap;
if(!Activities.empty()){ // Ci assicuriamo che la mappa sia piena
int N = Activities.size();
for(int turnaround = 1; turnaround<= N; turnaround++){
it_ActivitiesMap = Activities.begin(); // Si posiziona sul primo elemento della mappa
int i = 1;
for ( it_ActivitiesMap = Activities.begin(); i<=turnaround; it_ActivitiesMap++ ){
A.push_back(it_ActivitiesMap->second);
i++;
}
}
}
return(A);
}
pair<double, vector<int>> selectBestChromosome(multimap<double,vector<int>> chromosomes){
int n = chromosomes.size();
vector<int> numbers = vector_random(n);
int pos_random = Randint(0,n-2);
int max = numbers.at(pos_random);
if(max < numbers.at(pos_random+1))
max = numbers.at(pos_random+1);
multimap<double, vector<int>>::iterator it = chromosomes.begin();
for(int i = 0; i < max; i++)
it++;
return pair<double, vector<int>>((*it).first, (*it).second);
}
void update_rules(bool is_considered[])
{
/******************************/
/* Delete all the rules which
/* were considered by the
/* left factoring.*/
/******************************/
multimap<string,string>::iterator it;
for(int i=0;i<current_non_term_rules.size();i++)
{
if(!is_considered[i])
{
it=current_non_term_rules.begin();
advance(it,i);
left_factored_rules.insert(pair<string,string>((*it).first,(*it).second));
}
}
}
double getValue(){
double value = 0.0;
if(type == Filter_Average){
vector<double>::iterator kernelIter = kernel.begin();
vector<double>::iterator dataIter = data.end();
for(int i=0; i < data.size(); i++){
double datam = data[i];
if(kernel.size() != 0){
value += datam * (*kernelIter);
kernelIter++;
if(kernelIter == kernel.end()){
kernelIter = kernel.begin();
}
}
else{
value += datam;
}
dataIter--;
}
if(type==Filter_Average && kernel.size() == 0){
value = value / (double) bufferLength;
}
}
else if(type == Filter_Median){
//select median value
int middle = medianMap.size()/2;
map<double,double>::iterator medianIter = medianMap.begin();
for(int i=0; medianIter != medianMap.end(); medianIter++,i++){
if(i==middle){
value = medianIter->second;
break;
}
}
}
return value;
}
void TiKNeighborhood::verifyKCandidateNeighborsForward (
const vector<KNeighborhoodPoint>& dataset
, KNeighborhoodPoint& point
, vector<KNeighborhoodPoint>::iterator& pointForwardIt
, bool& forwardSearch
, multimap<double, vector<KNeighborhoodPoint>::iterator, DistanceComparator>& kNeighborhood
, unsigned long k
, unsigned long& realDistanceCalculationsCounter){
double distance;
unsigned long i;
while(forwardSearch && ((pointForwardIt->distance[0] - point.distance[0]) <= point.eps)){
distance = Point::minkowskiDistance((*pointForwardIt), point, 2);
realDistanceCalculationsCounter++;
if(distance < point.eps){
i = getKeysNr(kNeighborhood, point.eps);
if((kNeighborhood.size() - i) >= (k - 1)){
kNeighborhood.erase(point.eps);
kNeighborhood.insert(pair<double, vector<KNeighborhoodPoint>::iterator>(distance, pointForwardIt));
point.eps = getMaxDistance(kNeighborhood);
}
else{
kNeighborhood.insert(pair<double, vector<KNeighborhoodPoint>::iterator>(distance, pointForwardIt));
}
}
else
if(distance == point.eps){
kNeighborhood.insert(pair<double, vector<KNeighborhoodPoint>::iterator>(distance, pointForwardIt));
}
forwardSearch = followingPoint(dataset, pointForwardIt);
}
}
multimap<double, vector<int>> cross(multimap<double, vector<int>> chromosomes, double probability, vector<Data> train){
int n = chromosomes.size();
vector<int> order = vector_random(n);
vector<int> order_cross;
pair <pair<double,vector<int>>, pair<double,vector<int>>> chromosomes_songs;
multimap<double, vector<int>> chromosomed_crossed;
int n_cross = n*probability;
for(int i = 0; i < n_cross; i++){
order_cross.push_back(order.at(i));
}
for(int i = 0; i < order_cross.size()-1; i+=2) {
int c1 = order_cross.at(i);
int c2 = order_cross.at(i + 1);
multimap<double, vector<int>>::iterator it1 = chromosomes.begin(), it2 = chromosomes.begin();
for (int j = 0; j < c1; j++)
it1++;
for (int j = 0; j < c2; j++)
it2++;
chromosomes_songs = cross(*it1, *it2, train);
chromosomed_crossed.insert(chromosomes_songs.first);
chromosomed_crossed.insert(chromosomes_songs.second);
}
for(int i = n_cross-1; i < order.size(); i++){
int v = order.at(i);
multimap<double, vector<int>>::iterator it = chromosomes.begin();
for (int j = 0; j < v; j++)
it++;
chromosomed_crossed.insert(*it);
}
return chromosomed_crossed;
}
void left_factor()
{
/*************************************/
/* Algorithm
/* Generate all the prefixes of
all the rules.
/* Take each prefix one by one
/* Check if it is the common
/* prefix of atleast two of the
/* rules.
/* Update the longest common prefix.*/
/*************************************/
bool non_terminal_used;
string RHS;
string pre;
multimap<string,string>::iterator it;
string new_left_rule;
// for(int i=0;i<prefixes.size();i++)
// {
// cout<<"Prefix:"<<prefixes[i]<<endl;
// }
string prefix_of_rhs;
int len_of_prefix;
int len_of_longest_prefix;
string longest_prefix; // keeps track of what is the longest prefix.
int longest_prefix_index=-1;
int count_of_matches; // keeps track of how many rules match for a given prefix.
string left,remaining; // left holds the prefix. // remaining holds the remaining RHS.
string new_non_terminal;
for(int n=0;n<n_t.size();n++)
{
memset(is_considered,0,sizeof(is_considered));
memset(is_matched,0,sizeof(is_matched));
prefixes.clear();
current_non_term_rules.clear();
it=rules.find(n_t[n]);
for(;it!=rules.end() && ((*it).first.compare(n_t[n])==0);it++)
{
current_non_term_rules.insert(pair<string,string>((*it).first,(*it).second));
}
for(it=current_non_term_rules.begin();it!=current_non_term_rules.end();it++)
{
pre.clear();
RHS=(*it).second;
//cout<<(*it).first<<"-->"<<(*it).second<<endl;
//cout<<RHS.size();
for(int i=0;i<RHS.size();i++)
{
pre.clear();
for(int j=0;j<=i;j++)
{
pre.push_back(RHS[j]);
}
if(find(prefixes.begin(),prefixes.end(),pre)==prefixes.end())
prefixes.push_back(pre);
}
}
// for(it=current_non_term_rules.begin();it!=current_non_term_rules.end();it++)
// {
// cout<<(*it).first<<"-->"<<(*it).second;
// }
for(int m=0;m<current_non_term_rules.size();m++)
{
//cout<<"Round "<<m<<endl;
longest_prefix.clear();
for(int i=0;i<prefixes.size();i++)
{
//cout<<"Current Prefix:"<<prefixes[i]<<endl;
len_of_prefix=prefixes[i].size();
count_of_matches=0;
for(it=current_non_term_rules.begin();it!=current_non_term_rules.end();it++)
{
// if(is_matched[it-rules.begin()])
// continue;
prefix_of_rhs.clear();
RHS.clear();
RHS=(*it).second;
//cout<<(*it).first<<"-->"<<(*it).second<<endl;
if(RHS.size()<len_of_prefix|| is_considered[distance(current_non_term_rules.begin(),it)])
continue;
for(int j=0;j<len_of_prefix;j++)
{
prefix_of_rhs.push_back(RHS[j]);
}
//cout<<"Prefix of RHS:"<<prefix_of_rhs<<endl;
if(prefix_of_rhs.compare(prefixes[i])==0)
{
//cout<<prefix_of_rhs<<"matched with "<<prefixes[i]<<endl;
//cout<<"Rule No:"<<distance(rules.begin(),it)<<endl;
is_matched[i][distance(current_non_term_rules.begin(),it)]=true;
count_of_matches++;
}
}
// cout<<"RUles after longest prefix"<<endl;
// for(it=rules.begin();it!=rules.end();it++)
// {
// cout<<"Rule No:"<<<<"-"<<(*it).first<<"-->"<<(*it).second<<endl;
// }
//cout<<"Prefix is:"<<prefixes[i]<<" has count:"<<count_of_matches<<endl;
if(count_of_matches>1 && prefixes[i].size()>=1)
{
if(longest_prefix.size()<prefixes[i].size())
{
longest_prefix.clear();
longest_prefix=prefixes[i];
longest_prefix_index=i;
}
}
}
//cout<<"Index:"<<longest_prefix_index<<"and longest prefix is :"<<longest_prefix<<"!"<<endl;
if(longest_prefix_index!=-1 && longest_prefix.size()>=1)
{
/* Get rules which matched the longest prefix.*/
non_terminal_used=false;
new_non_terminal=next_nt_string();
for(int j=0;j<current_non_term_rules.size();j++)
{
if(is_matched[longest_prefix_index][j] && !is_considered[j])
{
non_terminal_used=true;
it=current_non_term_rules.begin();
advance(it,j);
RHS=(*it).second;
//cout<<"Rule No:"<<j<<"-"<<(*it).first<<"-->"<<(*it).second<<endl;
len_of_longest_prefix=longest_prefix.size();
left.clear();
remaining.clear();
for(int k=0;k<len_of_longest_prefix;k++)
{
left.push_back(RHS[k]);
}
if(RHS[len_of_longest_prefix]==' ')
len_of_longest_prefix++;
for(int k=len_of_longest_prefix;k<RHS.size();k++)
{
remaining.push_back(RHS[k]);
}
if(remaining.size()==0)
remaining="$"; //epsilon.
//cout<<"Remaining:"<<remaining<<" and new non terminal: "<<new_non_terminal<<endl;
/* Erase the current rule an push back the new rule in rules.*/
assert(left.compare(longest_prefix)==0);
//cout<<"Prefix:"<<left<<"and remaining: "<<remaining<<endl;
new_left_rule.clear();
if(longest_prefix[longest_prefix.size()-1]!=' ')
new_left_rule.append(longest_prefix).append(" ").append(new_non_terminal);
else
new_left_rule.append(longest_prefix).append(new_non_terminal);
left_factored_rules.insert(pair<string,string>(new_non_terminal,remaining));
is_considered[distance(current_non_term_rules.begin(),it)]=true;
//rules.erase(it);
}
}
if(non_terminal_used)
{
left_factored_rules.insert(pair<string,string>((*it).first,new_left_rule));
left_fact_n_t.push_back(new_non_terminal);
}
}
}
update_rules(is_considered);
}
multimap<string,string>::iterator iter;
for(iter=left_factored_rules.begin();iter!=left_factored_rules.end();iter++)
{
cout<<(*iter).first<<"-->"<<(*iter).second<<endl;
}
}
void GetSchoolKeyWord(const string &record,const string &raw_record,multimap<string::size_type,string> &m_keys, string &key_word,const string &last_key)
{
multimap<string::size_type,string>::size_type len = m_keys.size();
if(len == 0)
return;
string kword1,kword2;
string::size_type pos1 = 0,pos2 = 0;
if(last_key.compare("附属") == 0)
{
pos1 = record.find(last_key);
key_word = record.substr(pos1,record.size()-pos1);
return;
}
multimap<string::size_type,string>::iterator miter = m_keys.end();
if(last_key.compare("其它") == 0)
{
kword1 = (--miter)->second;
pos1 = record.rfind(kword1);
key_word = record.substr(pos1+kword1.size(),record.size()-pos1);
return;
}
multimap<string::size_type,string>::iterator miter1,miter2;
if(len >= 2)
{
miter2 = --miter;
miter1 = --miter2;
pos1 = record.find(miter1->second);
pos2 = record.find(miter2->second,pos1+(miter1->second).size());
key_word = record.substr(pos1+(miter1->second).size(),pos2-pos1);
return;
}
for(int i = 0; i < CAMPUSNUM; ++i)
{
if((pos1 = record.rfind(campus[i])) != string::npos)
{
kword1 = (--miter)->second;
if(len >= 2)
pos2 = record.rfind(kword1);
else
pos2 = record.find(kword1);
if(pos2 + kword1.size() != pos1)
key_word = record.substr(pos2+kword1.size(),record.size() - pos2 - kword1.size());
return;
}
}
string str;
if((pos1 = raw_record.find("(")) != string::npos)
{
if(pos2 = raw_record.find(")",pos1) != string::npos)
str = raw_record.substr(pos1+1,raw_record.size()-pos1-2);
else
str = raw_record.substr(pos1+1,raw_record.size()-pos1-1);
for(int i = 0; i < CAMPUSNUM; ++i)
{
string::size_type pos3 = str.find(campus[i]);
if(pos3 != string::npos && (pos3 + (campus[i]).size() == str.size()))
{
key_word = str;
return;
}
}
}
}
void CCrawl::Fetch(void *arg)
{
string str_url,host;
int nGsock = -1;//之前的套接字文件描述符
string strGHost;//之前的主机号
//生成一个PSE file来存放网页数据
//string ofs_name = DATA_PSE_FILE + "." + CStrFunction::itos(GetCurrentThreadId());//PSE.raw+当前线程号
string ofs_name = DATA_PSE_FILE + CStrFunction::itos(GetCurrentThreadId())+ ".txt";//PSE+当前线程号+.txt
CPSEFile pse_file(ofs_name);//创建一个PSE格式的文件，保存为原始网页库
//生成一个link_for_pse file来存放链接数据
ofs_name = DATA_LINK_FOR_PSE_FILE + CStrFunction::itos(GetCurrentThreadId())+ ".txt";//PSE+当前线程号+.txt
CLinkForPSEFile link_for_pse_file(ofs_name);//创建一个网页结构库
int isleep_cnt = 0;//线程运行控制参数
for(;;)
{
WaitForSingleObject(mutex_collection,INFINITE);//互斥锁
int cnt = map_urls.size();
if(cnt > 0)
{
//已经收集的没有访问的url
cout<<"collection has "<<cnt<<" unvisited urls"<<endl;
multimap<string,string>::iterator it = map_urls.begin();
if(it != map_urls.end())
{
//从带访问的url队列中得到一个url进行访问
str_url = (*it).second;
map_urls.erase(it);
ReleaseMutex(mutex_collection);
//分解url
CUrl iurl;
//看看url是否有http://，没有则返回
if(iurl.ParseUrl(str_url) == false)
{
cout<<"parse url false in Fetch"<<str_url<<endl;
continue;
}
//表明现在抓取的网页所在的主机，同之前抓取的网页所在的主机不同
//我们不能利用之前的套接字文件描述符进行CS通信,必须创建新的
//套接字文件描述符进行通信,这是由于循环导致的
if(strGHost != iurl.host_name)
{
closesocket(nGsock);
nGsock = -1;
strGHost = iurl.host_name;
}
//根据URL以及套接字文件描述符抓取URL对应的网页,并保存为原始网页库和网页结构库
((CCrawl *)arg)->DownroadFile(&pse_file,&link_for_pse_file,iurl,nGsock);
cnt = 0;
}else
{
ReleaseMutex(mutex_collection);
}
}else
{
//等待访问的url队列map_urls已经没有url了，这是我们需要挂起线程进行等待
ReleaseMutex(mutex_collection);
Sleep(1000);
isleep_cnt++;
}
if(b_f_over == true && isleep_cnt == 200)//当线程挂起的次数达到两百的时候，结束调用fetch
{
break;
}
}
pse_file.Close();
link_for_pse_file.Close();
}
int MarkovPhraseCreation::CreateWordPhrases()
{
try
{
if (n == 1)
PrintChildWordPhrases(feed); // Special case n = 1. Unigram phrases has no state awareness.
else
{
// Iterate through the most common starts
__int64 progress = 0;
multimap<int, string>::reverse_iterator nextstartstate;
for (nextstartstate = startngrams.rbegin(); nextstartstate != startngrams.rend(); ++nextstartstate)
{
string startphrase = feed;
startphrase.append(nextstartstate->second);
string startstate = nextstartstate->second;
// Prepare the start phrase by removing BREAKs and whitespaces
startphrase.erase(remove(startphrase.begin(), startphrase.end(), BREAK[0]), startphrase.end());
startphrase.erase(remove(startphrase.begin(), startphrase.end(), ' '), startphrase.end());
// And the start state by using the n-1 last words
int statebreakpos = nextstartstate->second.size() - 1;
for (int i = 0; i < n - 1; i++)
statebreakpos = nextstartstate->second.substr(0, statebreakpos).find_last_of(' ');
startstate = nextstartstate->second.substr(statebreakpos + 1);
// Start the recursive chain to print all phrases based on this start
PrintChildWordPhrases(startphrase, startstate, n-1);
progress++;
time_t progresstime = time(NULL);
clog << "Done with phrases starting with \"" << startphrase << "\". About " << progress * 100 / startngrams.size() << "% done at " << ctime(&progresstime);
}
}
}
catch (const exception& e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
int MarkovPhraseCreation::CreateCharPhrases(){
try
{
if (n == 1)
PrintChildCharPhrases(feed); // Special case n = 1. Unigram phrases has no state awareness.
else
{
// Iterate through the most common starts
__int64 progress = 0;
multimap<int, string>::reverse_iterator nextstartstate;
for (nextstartstate = startngrams.rbegin(); nextstartstate != startngrams.rend(); ++nextstartstate)
{
string startfeed = (feed.length() >= n) ? feed : nextstartstate->second;
int feedwords = count(startfeed.begin(), startfeed.end(), SPACE[0]);
startfeed.erase(remove(startfeed.begin(), startfeed.end(), BREAK[0]), startfeed.end());
startfeed.erase(remove(startfeed.begin(), startfeed.end(), SPACE[0]), startfeed.end());
PrintChildCharPhrases(startfeed, nextstartstate->second.substr(nextstartstate->second.length() - (n - 1)), 1+feedwords);
progress++;
time_t progresstime = time(NULL);
clog << "Done with phrases starting with \"" << startfeed << "\". About " << progress*100/startngrams.size() << "% done at " << ctime(&progresstime);
}
}
}
catch (const exception& e) {
cerr << e.what() << endl;
return 1;
}
return 0;
}
void dump_weigths(const multimap<int32_t, ServerCollect*>& weights)
{
printf("-----------------------dump_weigths(%"PRI64_PREFIX"u)----------------------\n", weights.size());
multimap<int32_t, ServerCollect*>::const_iterator it = weights.begin();
while (it != weights.end())
{
printf("server(%s), weight(%d)", tbsys::CNetUtil::addrToString(it->second->get_ds()->id_).c_str(), it->first);
++it;
}
printf("-----------------------dump_weigths end----------------------\n");
}
static CRF_State
crfstate(const vector<Token> &vt, int i)
{
CRF_State sample;
string str = vt[i].str;
// string str = normalize(vt[i].str);
sample.label = vt[i].pos;
sample.add_feature("W0_" + vt[i].str);
sample.add_feature("NW0_" + normalize(str));
string prestr = "BOS";
if (i > 0) prestr = vt[i-1].str;
// if (i > 0) prestr = normalize(vt[i-1].str);
string prestr2 = "BOS";
if (i > 1) prestr2 = vt[i-2].str;
// if (i > 1) prestr2 = normalize(vt[i-2].str);
string poststr = "EOS";
if (i < (int)vt.size()-1) poststr = vt[i+1].str;
// if (i < (int)vt.size()-1) poststr = normalize(vt[i+1].str);
string poststr2 = "EOS";
if (i < (int)vt.size()-2) poststr2 = vt[i+2].str;
// if (i < (int)vt.size()-2) poststr2 = normalize(vt[i+2].str);
sample.add_feature("W-1_" + prestr);
sample.add_feature("W+1_" + poststr);
sample.add_feature("W-2_" + prestr2);
sample.add_feature("W+2_" + poststr2);
sample.add_feature("W-10_" + prestr + "_" + str);
sample.add_feature("W0+1_" + str + "_" + poststr);
sample.add_feature("W-1+1_" + prestr + "_" + poststr);
//sample.add_feature("W-10+1_" + prestr + "_" + str + "_" + poststr);
// sample.add_feature("W-2-1_" + prestr2 + "_" + prestr);
// sample.add_feature("W+1+2_" + poststr + "_" + poststr2);
// train = 10000 no effect
// if (i > 0 && prestr.size() >= 3)
// sample.add_feature("W-1S_" + prestr.substr(prestr.size()-3));
// if (i < (int)vt.size()-1 && poststr.size() >= 3)
// sample.add_feature("W+1S_" + poststr.substr(poststr.size()-3));
// sentence type
// sample.add_feature("ST_" + vt[vt.size()-1].str);
for (size_t j = 1; j <= 10; j++) {
char buf[1000];
// if (str.size() > j+1) {
if (str.size() >= j) {
sprintf(buf, "SUF%d_%s", (int)j, str.substr(str.size() - j).c_str());
sample.add_feature(buf);
}
// if (str.size() > j+1) {
if (str.size() >= j) {
sprintf(buf, "PRE%d_%s", (int)j, str.substr(0, j).c_str());
sample.add_feature(buf);
}
}
for (size_t j = 0; j < str.size(); j++) {
if (isdigit(str[j])) {
sample.add_feature("CTN_NUM");
break;
}
}
for (size_t j = 0; j < str.size(); j++) {
if (isupper(str[j])) {
sample.add_feature("CTN_UPP");
break;
}
}
for (size_t j = 0; j < str.size(); j++) {
if (str[j] == '-') {
sample.add_feature("CTN_HPN");
break;
}
}
bool allupper = true;
for (size_t j = 0; j < str.size(); j++) {
if (!isupper(str[j])) {
allupper = false;
break;
}
}
if (allupper) sample.add_feature("ALL_UPP");
if (WNdic.size() > 0) {
const string n = normalize(str);
for (map<string, string>::const_iterator i = WNdic.lower_bound(n); i != WNdic.upper_bound(n); i++) {
sample.add_feature("WN_" + i->second);
}
}
// for (int j = 0; j < vt.size(); j++)
// cout << vt[j].str << " ";
// cout << endl;
// cout << i << endl;
// cout << sample.label << "\t";
// for (vector<string>::const_iterator j = sample.features.begin(); j != sample.features.end(); j++) {
// cout << *j << " ";
// }
// cout << endl;
return sample;
}
void testOneFunction( std::string funcParamName,
vector<string> argvList,
bool debug, int nrOfNodes,
multimap <string, int> results,
multimap <string, int> useresults) {
if (debug)
cout <<"\n\n------------------------------------------\ntesting ... " << argvList[1] << endl;
// Build the AST used by ROSE
SgProject* project = frontend(argvList);
// Call the Def-Use Analysis
DFAnalysis* defuse = new DefUseAnalysis(project);
int val = defuse->run(debug);
if (debug)
std::cout << "Analysis run is : " << (val ? "failure" : "success" ) << " " << val << std::endl;
if (val==1) exit(1);
if (debug==false)
defuse->dfaToDOT();
//std::list<SgNode*> vars = NodeQuery::querySubTree(project, V_SgFunctionDefinition);
//std::list<SgNode*>::const_iterator i = vars.begin();
NodeQuerySynthesizedAttributeType vars = NodeQuery::querySubTree(project, V_SgFunctionDefinition);
NodeQuerySynthesizedAttributeType::const_iterator i = vars.begin();
for (; i!=vars.end();++i) {
SgFunctionDefinition* func = isSgFunctionDefinition(*i);
std::string name = func->class_name();
string funcName = func->get_declaration()->get_qualified_name().str();
int maxNodes = defuse->getDefSize();
int nodeNr = defuse->getIntForSgNode(func);
if (nodeNr == -1)
continue;
//cout << " checking function : " << funcName << endl;
if (funcName!=funcParamName)
continue;
if (debug)
cout << "\n------------------------\nchecking for " << name << " -- " << funcName << " -- " << nodeNr << endl;
if (maxNodes!=nrOfNodes) {
cerr << " Error: Test should have " << nrOfNodes << " nodes. found: " << maxNodes << endl;
abort();
}
if (debug)
cout << " Test has nodes: " << nrOfNodes << endl;
if (debug)
cout <<"\nChecking all definitions ... " << endl;
// check nodes in multimap
std::vector <std::pair < SgInitializedName*, SgNode*> > map = defuse->getDefMultiMapFor(func);
if (map.size()>0) {
std::vector < std::pair <SgInitializedName*, SgNode*> >::const_iterator j = map.begin();
unsigned int hit=0;
SgNode* node = NULL;
string name="";
for (;j!=map.end();++j) {
SgInitializedName* in_node = j->first;
node = j->second;
name= in_node->get_qualified_name().str();
if (debug)
cout << " ... checking : " << name << endl;
multimap <string, int>::const_iterator k =results.begin();
for (;k!=results.end();++k) {
string resName = k->first;
int resNr = k->second;
int tableNr = defuse->getIntForSgNode(node);
if (name==resName)
if (debug)
cout << " ... defNr: " << resNr << " inTable: " << tableNr << endl;
if (name==resName && tableNr==resNr) {
hit++;
if (debug)
cout << " Hit " << hit << "/" << results.size() << " - (" << name << "," << resNr << ")" << endl;
}
}
}
if (hit!=results.size()) {
cerr << " Error: No hit! ... DFA values of node " << nodeNr << " are not correct! " << endl;
exit(1);
}
} else {
if (results.size()!=0) {
cerr << " Error: Test node " << defuse->getIntForSgNode(func) << " should have a multimap. " << endl;
exit(1);
}
}
if (debug)
cout <<"\nChecking all uses ... " << endl;
// check nodes in multimap
map = defuse->getUseMultiMapFor(func);
if (map.size()>0) {
std::vector <std::pair <SgInitializedName*, SgNode*> >::const_iterator j = map.begin();
size_t hit=0;
for (;j!=map.end();++j) {
SgInitializedName* in_node = j->first;
SgNode* node = j->second;
string name= in_node->get_qualified_name().str();
if (debug)
cout << " ... checking : " << name << endl;
multimap <string, int>::const_iterator k =useresults.begin();
for (;k!=useresults.end();++k) {
string resName = k->first;
int resNr = k->second;
int tableNr = defuse->getIntForSgNode(node);
if (name==resName)
if (debug)
cout << " ... defNr: " << resNr << " inTable: " << tableNr << endl;
if (name==resName && tableNr==resNr) {
hit++;
if (debug)
cout << " Hit " << hit << "/" << useresults.size() << " - (" << name << "," << resNr << ")" << endl;
}
}
}
if (hit!=useresults.size()) {
cerr << " Error: No hit! ... DFA values of node " << nrOfNodes << " are not correct! " << endl;
exit(1);
}
} // if
}
if (debug)
std::cout << "Analysis test is success." << std::endl;
}
void testOneFunction( std::string funcParamName,
vector<string> argvList,
bool debug, int nrOfNodes,
multimap <int, vector<string> > resultsIn,
multimap <int, vector<string> > resultsOut) {
cout << " \n\n------------------------------------------\nrunning (variable)... " << argvList[1] << endl;
// Build the AST used by ROSE
SgProject* project = frontend(argvList);
// Call the Def-Use Analysis
DFAnalysis* defuse = new DefUseAnalysis(project);
int val = defuse->run(debug);
if (debug)
std::cerr << ">Analysis run is : " << (val ? "failure" : "success" ) << " " << val << std::endl;
if (val==1) exit(1);
if (debug==false)
defuse->dfaToDOT();
LivenessAnalysis* liv = new LivenessAnalysis(debug,(DefUseAnalysis*)defuse);
std::vector <FilteredCFGNode < IsDFAFilter > > dfaFunctions;
NodeQuerySynthesizedAttributeType vars = NodeQuery::querySubTree(project, V_SgFunctionDefinition);
NodeQuerySynthesizedAttributeType::const_iterator i = vars.begin();
bool abortme=false;
int hitIn=0;
int hitOut=0;
for (; i!=vars.end();++i) {
SgFunctionDefinition* func = isSgFunctionDefinition(*i);
std::string name = func->class_name();
string funcName = func->get_declaration()->get_qualified_name().str();
if (debug)
cerr << " .. running live analysis for func : " << funcName << endl;
FilteredCFGNode <IsDFAFilter> rem_source = liv->run(func,abortme);
if (abortme)
break;
if (funcName!=funcParamName) {
if (debug)
cerr << " .. skipping live analysis check for func : " << funcName << endl;
continue;
}
if (rem_source.getNode()!=NULL)
dfaFunctions.push_back(rem_source);
NodeQuerySynthesizedAttributeType nodes = NodeQuery::querySubTree(func, V_SgNode);
// Edg3 mistakenly adds SgType nodes to the AST; Edg4 adds some also, but fewer. So we just remove them all. They
// make no difference in the variable-liveness analysis anyway.
nodes.erase(std::remove_if(nodes.begin(), nodes.end(), is_type_node), nodes.end());
SgFunctionDeclaration* decl = isSgFunctionDeclaration(func->get_declaration());
ROSE_ASSERT(decl);
Rose_STL_Container<SgInitializedName*> args = decl->get_parameterList()->get_args();
if (debug)
cerr <<"Found args : " << args.size() << endl;
Rose_STL_Container<SgInitializedName*>::const_iterator it = args.begin();
for (;it!=args.end();++it) {
nodes.push_back(*it);
}
if((int)nodes.size()-1!=nrOfNodes) {
cerr << "Error :: Number of nodes = " << nodes.size()-1 << " should be : " << nrOfNodes << endl;
exit(1);
} else {
if (debug)
cerr << "Investigating nodes : " << nodes.size() << endl;
}
NodeQuerySynthesizedAttributeType::const_iterator nodesIt = nodes.begin();
for (; nodesIt!=nodes.end();++nodesIt) {
SgNode* node = *nodesIt;
ROSE_ASSERT(node);
int tableNr = defuse->getIntForSgNode(node);
std::vector<SgInitializedName*> in = liv->getIn(node);
std::vector<SgInitializedName*> out = liv->getOut(node);
std::vector<string> inName;
std::vector<string> outName;
std::vector<SgInitializedName*>::const_iterator itv = in.begin();
for (;itv!=in.end();++itv) {
SgInitializedName* init = *itv;
string name = init->get_name();
inName.push_back(name);
}
itv = out.begin();
for (;itv!=out.end();++itv) {
SgInitializedName* init = *itv;
string name = init->get_name();
outName.push_back(name);
}
std::sort(inName.begin(), inName.end());
std::sort(outName.begin(), outName.end());
multimap <int, vector<string> >::const_iterator k =resultsIn.begin();
for (;k!=resultsIn.end();++k) {
int resNr = k->first;
vector<string> results = k->second;
if (debug)
cerr << " ... containing nodes : " << results.size() << " node: " << node->class_name()
<< " tableNr : " << tableNr
<< " resNr : " << resNr << endl;
if (tableNr==resNr) {
std::sort(results.begin(), results.end());
if (results==inName) {
if (debug)
cerr <<"Contents in IN vector is correct! " << endl;
} else {
if (debug) {
cerr << " >>>>>>>>>> Problem with contents for IN ! " << endl;
cerr << " >>>>>>>>>> RESULT ... " << endl;
}
std::vector<string>::const_iterator itv = inName.begin();
for (;itv!=inName.end();++itv) {
string name = *itv;
if (debug) cerr << name << " " ;
}
if (debug) {
cerr << endl;
cerr << " >>>>>>>>>> USER ... " << endl;
}
itv = results.begin();
for (;itv!=results.end();++itv) {
string name = *itv;
if (debug) cerr << name << " " ;
}
if (debug) {
cerr << endl;
}
exit(1);
}
if (results.size()==in.size()) {
hitIn++;
}
if (debug)
cout << " nodeNr: " << tableNr << ". ResultSize IN should be:" << results.size()
<< " - resultSize is: " << in.size()
<< " foundMatches == : " << hitIn << "/" << resultsIn.size() << endl;
}
}
k =resultsOut.begin();
for (;k!=resultsOut.end();++k) {
int resNr = k->first;
vector<string> results = k->second;
// cerr << " ... containing nodes : " << results.size() << " tableNr : " << tableNr <<
// " resNr : " << resNr << endl;
if (tableNr==resNr) {
std::sort(results.begin(), results.end());
if (results==outName) {
if (debug)
cerr <<"Contents in OUT vector is correct! " << endl;
} else {
if (debug) {
cerr << " >>>>>>>>>> Problem with contents for OUT ! " << endl;
cerr << " >>>>>>>>>> RESULT ... " << endl;
}
std::vector<string>::const_iterator itv = outName.begin();
for (;itv!=outName.end();++itv) {
string name = *itv;
if (debug) cerr << name << " " ;
}
if (debug) {
cerr << endl;
cerr << " >>>>>>>>>> USER ... " << endl;
}
itv = results.begin();
for (;itv!=results.end();++itv) {
string name = *itv;
if (debug)
cerr << name << " " ;
}
if (debug)
cerr << endl;
exit(1);
}
if (results.size()==out.size()) {
hitOut++;
}
if (debug)
cout << " nodeNr: " << tableNr << ". ResultSize OUT should be:" << results.size()
<< " - resultSize is: " << out.size()
<< " foundMatches == : " << hitOut << "/" << resultsOut.size() << endl;
}
}
if (hitIn==0 && hitOut==0) {
if (debug)
cout << " nodeNr: " << tableNr << " IN: " << hitIn << "/" << resultsIn.size() <<
" Out:" << hitOut << "/" << resultsOut.size() << endl;
}
}
if (hitIn!=(int)resultsIn.size() || hitOut!=(int)resultsOut.size()) {
cout << " Error: No hit! ... DFA values of node " << nrOfNodes << " are not correct! " << endl;
exit(1);
}
}
if (debug)
cerr << "Writing out to var.dot... " << endl;
std::ofstream f2("var.dot");
dfaToDot(f2, string("var"), dfaFunctions,
(DefUseAnalysis*)defuse, liv);
f2.close();
if (abortme) {
cerr<<"ABORTING ." << endl;
ROSE_ASSERT(false);
// exit(1);
}
// iterate again and write second var.dot file
i = vars.begin();
abortme=false;
std::vector <FilteredCFGNode < IsDFAFilter > > dfaFunctions2;
for (; i!=vars.end();++i) {
SgFunctionDefinition* func = isSgFunctionDefinition(*i);
std::string name = func->class_name();
string funcName = func->get_declaration()->get_qualified_name().str();
if (debug)
cerr << " .. running live analysis for func (fixupStatementsINOUT): " << funcName << endl;
FilteredCFGNode <IsDFAFilter> rem_source = liv->run(func,abortme);
liv->fixupStatementsINOUT(func);
if (rem_source.getNode()!=NULL)
dfaFunctions2.push_back(rem_source);
}
if (debug)
cerr << "Writing out to varFix.dot... " << endl;
std::ofstream f3("varFix.dot");
dfaToDot(f3, string("varFix"), dfaFunctions2,
(DefUseAnalysis*)defuse, liv);
f3.close();
if (debug)
std::cout << "Analysis test is success." << std::endl;
}
void InputHandler::initParamsFromDOM(DOMNode *node, multimap<string, ParamWrapper*> registry)
{
assert(node);
char *tmp = XMLString::transcode(node->getNodeName());
crusde_debug("%s, line: %d, initParamsFromDOM for node: %s.", __FILE__, __LINE__, tmp);
XMLString::release(&tmp);
DOMNode *child = node->getFirstChild();
DOMNamedNodeMap *attributes = NULL;
DOMNode *attr = NULL;
multimap<string, ParamWrapper*>::iterator key_iter;
pair< multimap<string, ParamWrapper*>::iterator, multimap<string, ParamWrapper*>::iterator > key_range;
list<string> params_set;
int count_keys(-1);
unsigned int count_set(0);
int empty_param(0);
while (child)
{
if( child->getNodeType() == DOMNode::ELEMENT_NODE)
{
attributes = child->getAttributes();
attr = attributes->getNamedItem(ATTR_name.xmlStr());
char* name = XMLString::transcode(attr->getNodeValue());
if( XMLString::compareIString(child->getNodeName(), TAG_parameter.xmlStr()) == 0 )
{
//get number of keys that equal name
count_keys = static_cast<int>( registry.count(string(name)) );
//if there is anything in the registry ...
if( count_keys > 0 )
{
//equal_range gives two results: an iterator to the first and last element with key==name
key_range = registry.equal_range(string(name));
//all keys have values that are adresses of double variables in the repsective plugins
//each of those variables now gets a value assigned. the same value.
for ( key_iter=key_range.first; key_iter != key_range.second; ++key_iter)
{
//get the value from the DOM
StrXML value(attributes->getNamedItem(ATTR_value.xmlStr())->getNodeValue());
//write it into the variable that's strored at key_iter->second
if( (key_iter->second)->isString() )
{
(key_iter->second)->setValue( value.cppStr() );
}
else if( (key_iter->second)->isDouble() )
{
(key_iter->second)->setValue( static_cast<double>( atof(value.cStr()) ) );
}
++count_set;
}
//memorize the key that was set only once
params_set.push_back(key_range.first->first);
}
else //if not found there might be a spelling mistake either in input, or plugin, or both :)
{
StrXML paramName(attributes->getNamedItem(ATTR_name.xmlStr())->getNodeValue());
crusde_warning("Parameter %s (coming from the experiment definition) not registered! Misspelled in XML file or Plug-in definition?",paramName.cStr());
}
}
XMLString::release(&name);
}
child = child->getNextSibling();
}
//two cases are possible for leftover parameters:
// i) they are optional parameters - check registry for that
// ii) they are unitialized - oh well, we gotta stop there
if(count_set < registry.size())
{
multimap<string, ParamWrapper*>::iterator map_iter = registry.begin();
list<string>::iterator found_iter;
string out_string("Error: Parameters that remain uninitialized: \n");
while (map_iter != registry.end())
{
if(!map_iter->second->isOptional())
{
found_iter = params_set.begin();
//as long as we're not at the end and could not find the parameter in the list of set
//parameters, continue looking.
while( found_iter != params_set.end() && (*found_iter).compare(map_iter->first) != 0 )
{
++found_iter;
}
//if we're at the end, the parameter was not in out list ... tell the user.
if(found_iter==params_set.end() )
{
++empty_param;
out_string.append("\t");
out_string.append(map_iter->first);
out_string.append("\n");
}
}
//get to the next unique value in the registry ... avoid printing multiple keys twice.
map_iter = ( registry.equal_range(map_iter->first) ).second;
}
if(empty_param > 0)
{
crusde_error("%s, Aborting.", out_string.c_str());
}
}
}
int main() {
srand(time(NULL));
vector<string> names{ "Aragorn", "Andrev", "Chashbr", "Kostya", "Max", "Ilya", "Thireon", "Kirkorov", "Frodo" };
//создаем овтобусный парк, 10 автобусов, первый мы инициализируем временным объектом Bus, все последующие инициализируем оператором копирования для временного Bus
vector<Bus> park(10, Bus(route.begin()));
for (size_t i = 0; i < park.size(); i++) {//инициализируем массив событий прихода автобусов-по сти расписание
int t = i * 30;
events.insert(pair<int, Event*>(t, new BusEvent(t, park[i])));
}
Passenger** p = new Passenger*[3];//Массив указателей на пассажиров
if (p){
for (int i = 0; i < 3; ++i){//создали 3 пассажиров и расставили их рандомно на остановках, присвоили рандомные имена из массива имен
p[i] = new Passenger(route.begin() + rand() % (route.size() - 1), names[rand() % (names.size() - 1)], rand() % route.size() + 1);
cout << "passenger " << p[i]->name << " come to " << p[i]->position->name << endl;
}
}
//собственно на данный момент на карусели вертятся одновременно3 пассажира, они меняют свои имена и станции но их всегда трое
while (events.size() > 0) {
// get event
Event *ev = events.begin()->second;
int Time = events.begin()->first;
//мы удаляем совершившиеся событие из очереди событий
events.erase(events.begin());
/*тут мы как-то обрабатываем свершившееся событие, кстати в момент обработки текущго события
в очередь бобавляется следующшее планируемое событие. Т.е одно событие произошло автобус пришел на оcтановку, мы удалили
событие из очереди, а тут же автобус запланировал себе приход на следующую остановку, и мы добавлили новое событие в очередь*/
ev->process();
/*В этих двух циклах выполняется следующее: мы перебираем массив пассажиров, для каждого отдельного пассажира мы проверяем, не сидит
ли он в автобусе p[i]->currentBus==NULL
если не сидит мы перебираем автобусы и смотрим какой автобус в данный момент стоит на сотановк, после чего садим пассажира в него*/
for (int i = 0; i < 3; ++i){
if (p[i] == NULL){// т.к пассажиры будут удаляться из массива, нам нужно проверять есть ли в текущем указателе пассажир, и если его нет мы должны егшо сделать
p[i] = new Passenger(route.begin() + rand() % (route.size() - 1), names[rand() % (names.size() - 1)], rand() % route.size() + 1);
cout << "passenger " << p[i]->name << " come to " << p[i]->position->name << endl;
}
if (p[i]->currentBus == NULL){
for (size_t j = 0; j < park.size(); ++j){//посадили пассажира на автобус
if (park[j].position == p[i]->position){
p[i]->currentBus = &park[j];
p[i]->position = p[i]->currentBus->position;
cout << "Passenger " << p[i]->name << " take a bus number " << park[j].number << " on station " << p[i]->position->name <<
" and need to pass " << p[i]->positionCounter << " stations" << endl;
break;
}
}
}
}
/*Мы проверяем каждого пассажира, если он сидит в автобусе, и его(пассажира) позиция не равняется текущей позиции автобуса,
значит автобус с остановки уже уехал, прихватив пассажира,
и мы меняем положение позиции пассажира, после чего делаем декремент от количеств остановок которое пассажир должен проехать*/
for (int i = 0; i < 3; ++i){
if (p[i]->currentBus != NULL && p[i]->position != p[i]->currentBus->position){
p[i]->position = p[i]->currentBus->position;
(p[i]->positionCounter)--;
if (p[i]->positionCounter == 0){//пассажир приехал - удаляем его из массива, и делаем указатель равным 0
cout << "Passenger " << p[i]->name << " was gone from bus number " << p[i]->currentBus->number << endl;
delete p[i];
p[i] = NULL;
}
}
}
// удаляем указатель на событие
delete ev;
int s = time(0);
while (s == time(0));
}
}
