float language_model::perplexity_per_word(const sentence& tokens) const
{
if (tokens.size() == 0)
throw language_model_exception{
"perplexity_per_word() called on empty sentence"};
return perplexity(tokens) / tokens.size();
}
/* This function1 searches for a string in a sentence and return true/false*/
bool search_string(string str, sentence sen, int& j) {
j=0;
while (j!=sen.size()) {
if (!sen[j].compare(str)) {
break;
}
j++;
}
if (j==sen.size()) return false; else return true;
}
bool model_morphodita_parsito::parse(sentence& s, const string& options, string& error) const {
error.clear();
if (!parser) return error.assign("No parser defined for the UDPipe model!"), false;
parser_cache* c = parser_caches.pop();
if (!c) c = new parser_cache();
int beam_search = 5;
if (!named_values::parse(options, c->options, error))
return false;
if (c->options.count("beam_search"))
if (!parse_int(c->options["beam_search"], "beam_search", beam_search, error))
return false;
c->tree.clear();
for (size_t i = 1; i < s.words.size(); i++) {
c->tree.add_node(s.words[i].form);
c->tree.nodes.back().lemma.assign(s.words[i].lemma);
c->tree.nodes.back().upostag.assign(s.words[i].upostag);
c->tree.nodes.back().xpostag.assign(s.words[i].xpostag);
c->tree.nodes.back().feats.assign(s.words[i].feats);
c->tree.nodes.back().deps.assign(s.words[i].deps);
c->tree.nodes.back().misc.assign(s.words[i].misc);
}
parser->parse(c->tree, beam_search);
for (size_t i = 1; i < s.words.size(); i++)
s.set_head(i, c->tree.nodes[i].head, c->tree.nodes[i].deprel);
parser_caches.push(c);
return true;
}
void do_add(vector<fd>& x, sentence line) {
fd temp;
int length=line.size();
temp.property=line[length-1];
temp.filename=line[length-2];
vector <int> index;
int minlength=10000000;
for(int i=0;i!=length-3;i++)
index.push_back(-1);
for(int j=1;j!=length-2;j++)
for(int i=0;i!=x.size();i++)
if ((!x[i].property.compare(line[j])) && (!x[i].filename.compare(line[length-2])))
index[j-1]=i;
for(int i=0;i!=length-3;i++) {
if (index[i]>-1)
if (minlength>x[index[i]].value.size())
minlength=x[index[i]].value.size();
}
for (int j=0;j!=minlength;j++) {
float value=0;
for (int i=0;i!=length-2;i++)
if (index[i]>0)
value+=x[index[i]].value[j];
temp.value.push_back(value);
}
x.push_back(temp);
}
/*This function prints out the parameter associated index in successfully read file */
void print_file_index_output(sentence s, map<string, vector<int> > x, sentence sf) {
size_t maxlen=0;
for(int i=0;i!=sf.size();i++)
maxlen=max(maxlen,sf[i].length());
cout << print_space(maxlen+11);
for(sentence::iterator aa=++s.begin(); aa!=s.end(); aa++)
cout << *aa << "\t";
cout << endl;
for(int j=0;j!=f.size();j++) {
cout << "File-> " << sf[j] << print_space(maxlen-sf[j].length()+1) <<": ";
for(sentence::iterator aa=++s.begin(); aa!=s.end(); aa++) {
cout << x[*aa][j] << "\t";
}
cout << endl;
}
}
vector<fd> convert_to_float(sentence s, map<string, vector<int> > x, sentence sf) {
vector<fd> datatemp;
int style,set;
if (!s[0].compare("s")) style=1; else style=2;
for(int i=1;i!=s.size();i++) {
fd temp;
switch (style) {
case 1: { // This is series case
set=0;
for(int j=0;j!=f.size(); j++) {
if (x[s[i]][j]) {
if (!set) {
temp.filename="Allfiles";
temp.property=s[i];
set=1;
}
for(int tt=1;tt!=f[j].size();tt++) {
float t=atof(&f[j][tt][x[s[i]][j]][0]);
temp.value.push_back(t);
}
}
// datatemp.push_back(temp);
}
datatemp.push_back(temp);
break;
}
case 2: { // This is parallel case.
for(int j=0;j!=f.size(); j++) {
if (x[s[i]][j]) {
temp.filename=sf[j];
temp.property=s[i];
temp.value.clear();
for(int tt=1;tt!=f[j].size();tt++) {
float t= 0.0;
if ((f[j][tt][x[s[i]][j]].compare("nan")))
t = atof(&f[j][tt][x[s[i]][j]][0]);
temp.value.push_back(t);
}
datatemp.push_back(temp);
}
else if (!s[i].compare("Step")) {
temp.filename=sf[j];
temp.property=s[i];
temp.value.clear();
for(int tt=1;tt!=f[j].size();tt++) {
float t=atof(&f[j][tt][x[s[i]][j]][0]);
temp.value.push_back(t);
}
datatemp.push_back(temp);
}
}
break;
}
}
}
return datatemp;
}
std::string to_string(const sentence & s)
{
std::string result;
result.reserve(sentence::max_length);
result += s.get_start_token();
result += s.talker();
result += s.tag();
for (auto const & data : s.get_data()) {
result += ",";
result += data;
}
result += s.get_end_token();
char buf[8];
snprintf(buf, sizeof(buf), "%02X", checksum(result.begin() + 1, result.end() - 1));
return result + buf;
}
/*This function replaces certain input keywords to their appropriate words found in lammps file*/
void transform_userinput(sentence& temp) {
if (temp.size()) {
for(int i=1;i!=temp.size();i++) {
if (!temp[i].compare("t")) temp[i].replace(0,temp[i].length(),"Temp");
if (!temp[i].compare("p")) temp[i].replace(0,temp[i].length(),"Press");
if (!temp[i].compare("v")) temp[i].replace(0,temp[i].length(),"Volume");
if (!temp[i].compare("vd")) temp[i].replace(0,temp[i].length(),"E_vdwl");
if (!temp[i].compare("te")) temp[i].replace(0,temp[i].length(),"TotEng");
if (!temp[i].compare("ke")) temp[i].replace(0,temp[i].length(),"KinEng");
if (!temp[i].compare("pe")) temp[i].replace(0,temp[i].length(),"PotEng");
if (!temp[i].compare("be")) temp[i].replace(0,temp[i].length(),"E_bond");
if (!temp[i].compare("ae")) temp[i].replace(0,temp[i].length(),"E_angle");
if (!temp[i].compare("de")) temp[i].replace(0,temp[i].length(),"E_dihed");
if (!temp[i].compare("ce")) temp[i].replace(0,temp[i].length(),"E_coul");
if (!temp[i].compare("le")) temp[i].replace(0,temp[i].length(),"E_long");
}
}
}
/* This function add array of strings (passed as string) to a sentence */
sentence add_strings(sentence& temp_entry, const string l) {
sentence temp=split(l);
s_type mm;
if (temp.size()!=0)
for (mm=0;mm!=temp.size();mm++)
temp_entry.push_back(temp[mm]);
else
return temp_entry;
return temp_entry;
}
bool model_morphodita_parsito::tokenizer_morphodita::next_sentence(sentence& s, string& error) {
s.clear();
error.clear();
if (tokenizer->next_sentence(&forms, nullptr)) {
for (size_t i = 0; i < forms.size(); i++)
splitter.append_token(forms[i], i+1 < forms.size() && forms[i+1].str == forms[i].str + forms[i].len ? "SpaceAfter=No" : "", s);
return true;
}
return false;
}
/* This function searches if the parameter is present in the lammps file and return a array of
associated with the parameter stored in string */
map<string, vector<int> > update2_userinput(sentence temp) {
if ((!temp[0].compare("s")) || (!temp[0].compare("p"))) {
s_type j;
int jtemp;
int notfind;
int notfindall=1;
map<string, vector<int> > ret;
sentence::iterator tb=temp.begin();
for(j=0;j!=f.size();j++)
ret[*tb].push_back(0);
tb++;
for(sentence::iterator aa=tb; aa!=temp.end(); aa++) {
notfind=1;
for(j=0;j!=f.size();j++) {
if (search_string(*aa,f[j][0],jtemp)) {
ret[*aa].push_back(jtemp);
notfind=0;
notfindall=0;
}
else
ret[*aa].push_back(0);
}
if (notfind) {
temp.erase(aa);
aa--;
}
}
if (notfindall) {
cerr << "\nDid not find any matching parameters in any input files, read successfully. Please check!!!\n\n";
exit(1);
}
return ret;
}
else {
cerr << "\nFirst string should be s (for series) or p (for parallel)\n\n" << endl;
exit(1);
}
}
float language_model::perplexity(const sentence& tokens) const
{
if (tokens.size() == 0)
throw language_model_exception{"perplexity() called on empty sentence"};
return std::pow(10.0f, -(log_prob(tokens) / tokens.size()));
}
void plot_xmgrace(sentence s, vector <fd> data) {
ofstream out("temp");
// Some string definitions
string s1 = "@ s";
string s2 = " comment ";
string s3 = " legend ";
int plotiterator=0; int set; int style;
int half_data=data.size()/2;
// Printing out X and Y axis properties
out << "@ xaxis label char size 1.500000\n";
out << "@ xaxis ticklabel char size 1.250000\n";
if (s[1].compare("cross"))
out << "@ xaxis label " << "\"time\""<< endl;
else
out << "@ xaxis label " << "\"" << s[2] << "\""<< endl;
out << "@ yaxis label char size 1.500000\n";
out << "@ yaxis ticklabel char size 1.250000\n";
// Printing out legend properties
out << "@ legend box linestyle 0\n";
out << "@ legend 0.25, 0.8\n";
// Selecting style for series or parallel plotting.
if (!s[0].compare("s")) style=1; else style=2;
if (s[1].compare("cross")) {
if (s.size()==2) {
out << "@ yaxis label " << "\"";
for(int i=1;i!=s.size();i++)
out << s[i] << " ";
out << "\""<< endl;
}
}
else
out << "@ yaxis label " << "\"" << s[3] << "\""<< endl;
int length=data.size();
if (!s[1].compare("cross")) length/=2;
// Storing data in file "temp" as asked (series or parallel)
for(int i=0;i!=length;i++) {
switch (style) {
case 1: { // Series Case
if (s[1].compare("cross")) { // This is executed if not cross
out << s1 << plotiterator << s2 << "\""<< data[i].property << "\"" << endl;
out << s1 << plotiterator << s3 << "\""<< data[i].property << "\"" << endl;
}
else
{
out << s1 << plotiterator << s2 << "\""<< data[i+half_data].property << " vs " << data[i].property << "\"" << endl;
out << s1 << plotiterator << s3 << "\""<< data[i+half_data].property << " vs " << data[i].property << "\"" << endl;
}
out << s1 << plotiterator << " symbol 1" << endl;
out << s1 << plotiterator << " symbol size 0.50000" << endl;
out << s1 << plotiterator << " symbol color 1" << endl;
out << s1 << plotiterator << " symbol pattern 1" << endl;
out << s1 << plotiterator << " symbol fill color "<< plotiterator+1 << endl;
out << s1 << plotiterator << " symbol fill pattern 1" << endl;
out << s1 << plotiterator << " symbol linewidth 1.0" << endl;
out << s1 << plotiterator << " symbol linestyle 1" << endl;
out << s1 << plotiterator << " symbol char 65" << endl;
out << s1 << plotiterator << " symbol char font 0" << endl;
plotiterator++;
for(int j=0;j!=data[i].value.size(); j++)
if (s[1].compare("cross"))
out << data[i].value[j] << endl; // This is executed if not cross
else
out << data[i].value[j] << " " << data[i+half_data].value[j]<< endl; // This is executed if cross
out << "&" << endl;
break;
}
case 2: { // Parallel Case
if (s[1].compare("cross")) {
out << s1 << plotiterator << s2 << "\""<< data[i].property << ": " << data[i].filename << "\"" << endl;
if (s.size()<=2)
out << s1 << plotiterator << s3 << "\"" << data[i].filename << "\"" << endl;
else
out << s1 << plotiterator << s3 << "\""<< data[i].property << ": " << data[i].filename << "\"" << endl;
}
else {
out << s1 << plotiterator << s2 << "\""<< data[i+half_data].property << " vs " << data[i].property << ": " << data[i].filename << "\"" << endl;
out << s1 << plotiterator << s3 << "\""<< data[i+half_data].property << " vs " << data[i].property << ": " << data[i].filename << "\"" << endl;
}
out << s1 << plotiterator << " symbol 0" << endl;
out << s1 << plotiterator << " symbol size 0.50000" << endl;
out << s1 << plotiterator << " symbol color 1" << endl;
out << s1 << plotiterator << " symbol pattern 1" << endl;
out << s1 << plotiterator << " symbol fill color "<< plotiterator+1 << endl;
out << s1 << plotiterator << " symbol fill pattern 1" << endl;
out << s1 << plotiterator << " symbol linewidth 1.0" << endl;
out << s1 << plotiterator << " symbol linestyle 1" << endl;
out << s1 << plotiterator << " symbol char 65" << endl;
out << s1 << plotiterator << " symbol char font 0" << endl;
plotiterator++;
for(int j=0;j!=data[i].value.size(); j++)
if (s[1].compare("cross")) {
out << data[i].value[j] << endl; // This is executed if not cross
}
else
out << data[i].value[j] << " " << data[i+half_data].value[j]<< endl; // This is executed if cross
out << "&" << endl;
break;
}
}
}
int rv=system("xmgrace temp &");
}
/*This function reads the data from the input files*/
void read_input(int argc, char** argv, sentence& sf,string& analysisfile) {
filedata a;
int success_filecounter=0;
int leave_index=0;
// Read the data into the file.
for (int i=1; i<argc; ++i) {
if (!strcmp(argv[i],"-leave")) {
leave_index=atoi(argv[i+1]);
i++;
continue;
}
if (!strcmp(argv[i],"-analysis")) {
analysisfile=argv[i+1];
i++;
continue;
}
ifstream in(argv[i]);
if (in) {
sf.push_back(argv[i]);
cout << "Reading file : " << argv[i] << endl;
if (a.size())
a.clear();
readlogfile_multi(a,in,leave_index);
f.push_back(a);
in.close();
success_filecounter++;
} else {
cerr << "Cannot open file: " << argv[i] << endl;
}
}
if(!success_filecounter)
exit(1);
else {
sentence aa;
int exists;
cout << endl;
cout << "Select parameters to plot from following: --- " << endl;
for (int i=0;i<success_filecounter;i++) {
for(int j=0;j<f[i][0].size();j++) {
exists=0;
for(int k=0;k!=aa.size();k++)
if (!f[i][0][j].compare(aa[k])) {
exists=1;
break;
}
if (!exists)
aa.push_back(f[i][0][j]);
}
}
for(int k=0;k!=aa.size();k++) {
if (!aa[k].compare("Temp")) aa[k].replace(0,aa[k].length(),"Temp(t)");
if (!aa[k].compare("Press")) aa[k].replace(0,aa[k].length(),"Press(p)");
if (!aa[k].compare("Volume")) aa[k].replace(0,aa[k].length(),"Volume(v)");
if (!aa[k].compare("E_vdwl")) aa[k].replace(0,aa[k].length(),"E_vdwl(vd)");
if (!aa[k].compare("TotEng")) aa[k].replace(0,aa[k].length(),"TotEng(te)");
if (!aa[k].compare("KinEng")) aa[k].replace(0,aa[k].length(),"KinEng(ke)");
if (!aa[k].compare("PotEng")) aa[k].replace(0,aa[k].length(),"PotEng(pe)");
if (!aa[k].compare("E_bond")) aa[k].replace(0,aa[k].length(),"E_bond(be)");
if (!aa[k].compare("E_angle")) aa[k].replace(0,aa[k].length(),"E_angle(ae)");
if (!aa[k].compare("E_dihed")) aa[k].replace(0,aa[k].length(),"E_dihed(de)");
if (!aa[k].compare("E_coul")) aa[k].replace(0,aa[k].length(),"E_coul(ce)");
if (!aa[k].compare("E_long")) aa[k].replace(0,aa[k].length(),"E_long(le)");
cout << aa[k];
if (aa[k].size()<8) cout << "\t";
cout << "\t";
if (k%6==5) cout << endl;
}
cout << "\n--------------\n\n";
cout << "Please enter your parameters in the following format: --- " << endl;
cout << "1. First string should be 'p' or 's' for plotting parameters from different files in parallel or series" << endl;
cout << "2. Please enter one or more parameters listed above after entering the first string." << endl;
cout << "3. For certain parameters, short forms may be used as listed in bracket above" << endl;
cout << " ----- Example: To plot total energy, please type 'p te'." << endl;
cout << "4. In order to plot one parameter with respect to other, please type 'cross' after p/s followed by two parameters" << endl;
cout << "5. The independent parameter could be 'Step' as mentioned above." << endl;
cout << "6. Currently, only two parameters can be plotted with respect to each other." << endl;
cout << " ----- Example: To plot temperature vs. pressure p cross t p" << endl ;
cout << "--------------\n\n";
cout << "For analysis, please exit and use -analysis flag with filename. The file should contain atleast one of these commands: --- " << endl;
cout << "1. average all/single_filename startvalue(start) endvalue(end)" << endl;
cout << "2. scale filename property factor" << endl;
cout << "3. inverse filename property" << endl;
cout << "4. write Under Coding" << endl;
cout << "5. subtract property1 property2 filename outputpropertyname" << endl;
cout << "6. add Under Coding" << endl;
cout << "--------------\n\n";
}
}
/// Returns the ID of the manufacturer of the specified sentence.
///
/// @param[in] s Sentence to get the manufacturer ID from.
/// @return The manufacturers ID.
manufacturer_id get_manufacturer_id(const sentence & s)
{
return get_manufacturer_id(s.tag());
}
