void CSimulation2Impl::DumpState()
{
PROFILE("DumpState");
std::stringstream pid;
pid << getpid();
std::stringstream name;\
name << std::setw(5) << std::setfill('0') << m_TurnNumber << ".txt";
OsPath path = psLogDir() / "sim_log" / pid.str() / name.str();
CreateDirectories(path.Parent(), 0700);
std::ofstream file (OsString(path).c_str(), std::ofstream::out | std::ofstream::trunc);
file << "State hash: " << std::hex;
std::string hashRaw;
m_ComponentManager.ComputeStateHash(hashRaw, false);
for (size_t i = 0; i < hashRaw.size(); ++i)
file << std::setfill('0') << std::setw(2) << (int)(unsigned char)hashRaw[i];
file << std::dec << "\n";
file << "\n";
m_ComponentManager.DumpDebugState(file, true);
std::ofstream binfile (OsString(path.ChangeExtension(L".dat")).c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
m_ComponentManager.SerializeState(binfile);
}
void StelLocationMgr::generateBinaryLocationFile(const QString& fileName, bool isUserLocation, const QString& binFilePath) const
{
const QMap<QString, StelLocation>& cities = loadCities(fileName, isUserLocation);
QFile binfile(binFilePath);
binfile.open(QIODevice::WriteOnly);
QDataStream out(&binfile);
out.setVersion(QDataStream::Qt_4_6);
out << cities;
binfile.close();
}
QHash<int, QVector<int> > load_user_negatives(QString file_name, bool verbose)
{
QHash<int, QVector<int> > user_negatives;
if (verbose) printf("Start loading set from %s ... ", qPrintable(file_name));
QFile binfile(file_name + ".bin");
if (!binfile.exists())
{
QFile file(file_name + ".txt");
file.open(QFile::ReadOnly);
QTextStream in(&file);
int u = 0, i = 0;
QStringList list;
while (!in.atEnd())
{
QString line = in.readLine();
// printf(qPrintable(line+'\n'));
if (line.contains('|'))
{
list = line.split('|');
u = list.at(0).toInt();
QVector<int> items;
user_negatives.insert(u, items);
}
else
{
list = line.split('\t');
i = list.at(0).toInt();
user_negatives[u].append(i);
}
}
file.close();
// serializationRsHash
binfile.open(QFile::WriteOnly);
QDataStream bin(&binfile);
bin << user_negatives;
binfile.close();
}
else
{
// deserialization
binfile.open(QFile::ReadOnly);
QDataStream bin(&binfile);
bin >> user_negatives;
binfile.close();
}
if (verbose) printf("OK\n");
return user_negatives;
}
void save_factors() {
printf("Saving factors to files... ");
std::stringstream ss;
ss<<"../../user_factors_st="<<steps<<"_fn="<<fact_n<<"_a="<<alfa<<"_l="<<lambda;
QString file_name = QString::fromStdString(ss.str());
QFile binfile(file_name + ".bin");
binfile.open(QFile::WriteOnly);
QDataStream bin(&binfile);
bin << user_factors;
binfile.close();
file_name = file_name.replace("user", "item");
QFile binfile2(file_name + ".bin");
binfile2.open(QFile::WriteOnly);
QDataStream bin2(&binfile2);
bin2 << item_factors;
binfile2.close();
printf("ok\n");
}
void createProbingPT(const char * phrasetable_path, const char * target_path){
//Get basepath and create directory if missing
std::string basepath(target_path);
mkdir(basepath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
//Set up huffman and serialize decoder maps.
Huffman huffmanEncoder(phrasetable_path); //initialize
huffmanEncoder.assign_values();
huffmanEncoder.produce_lookups();
huffmanEncoder.serialize_maps(target_path);
//Get uniq lines:
unsigned long uniq_entries = huffmanEncoder.getUniqLines();
//Source phrase vocabids
std::map<uint64_t, std::string> source_vocabids;
//Read the file
util::FilePiece filein(phrasetable_path);
//Init the probing hash table
size_t size = Table::Size(uniq_entries, 1.2);
char * mem = new char[size];
memset(mem, 0, size);
Table table(mem, size);
BinaryFileWriter binfile(basepath); //Init the binary file writer.
line_text prev_line; //Check if the source phrase of the previous line is the same
//Keep track of the size of each group of target phrases
uint64_t entrystartidx = 0;
//uint64_t line_num = 0;
//Read everything and processs
while(true){
try {
//Process line read
line_text line;
line = splitLine(filein.ReadLine());
//Add source phrases to vocabularyIDs
add_to_map(&source_vocabids, line.source_phrase);
if ((binfile.dist_from_start + binfile.extra_counter) == 0) {
prev_line = line; //For the first iteration assume the previous line is
} //The same as this one.
if (line.source_phrase != prev_line.source_phrase){
//Create a new entry even
//Create an entry for the previous source phrase:
Entry pesho;
pesho.value = entrystartidx;
//The key is the sum of hashes of individual words. Probably not entirerly correct, but fast
pesho.key = 0;
std::vector<uint64_t> vocabid_source = getVocabIDs(prev_line.source_phrase);
for (int i = 0; i < vocabid_source.size(); i++){
pesho.key += vocabid_source[i];
}
pesho.bytes_toread = binfile.dist_from_start + binfile.extra_counter - entrystartidx;
//Put into table
table.Insert(pesho);
entrystartidx = binfile.dist_from_start + binfile.extra_counter; //Designate start idx for new entry
//Encode a line and write it to disk.
std::vector<unsigned char> encoded_line = huffmanEncoder.full_encode_line(line);
binfile.write(&encoded_line);
//Set prevLine
prev_line = line;
} else{
//If we still have the same line, just append to it:
std::vector<unsigned char> encoded_line = huffmanEncoder.full_encode_line(line);
binfile.write(&encoded_line);
}
} catch (util::EndOfFileException e){
std::cerr << "Reading phrase table finished, writing remaining files to disk." << std::endl;
binfile.flush();
//After the final entry is constructed we need to add it to the phrase_table
//Create an entry for the previous source phrase:
Entry pesho;
pesho.value = entrystartidx;
//The key is the sum of hashes of individual words. Probably not entirerly correct, but fast
pesho.key = 0;
std::vector<uint64_t> vocabid_source = getVocabIDs(prev_line.source_phrase);
for (int i = 0; i < vocabid_source.size(); i++){
pesho.key += vocabid_source[i];
}
pesho.bytes_toread = binfile.dist_from_start + binfile.extra_counter - entrystartidx;
//Put into table
table.Insert(pesho);
break;
}
}
serialize_table(mem, size, (basepath + "/probing_hash.dat").c_str());
serialize_map(&source_vocabids, (basepath + "/source_vocabids").c_str());
delete[] mem;
//Write configfile
std::ofstream configfile;
configfile.open((basepath + "/config").c_str());
configfile << uniq_entries << '\n';
configfile.close();
}
