int ReadRealData(const std::string dataPath, Cpu::real_vector &data)
{
//
std::ifstream ifs(dataPath.c_str(), std::ifstream::binary | std::ifstream::in);
if (!ifs.good())
throw std::runtime_error(std::string("Fail to open ")+dataPath);
// get the number of we data
std::streampos numEleS, numEleE;
long int numEle;
numEleS = ifs.tellg();
ifs.seekg(0, std::ios::end);
numEleE = ifs.tellg();
numEle = (numEleE-numEleS)/sizeof(real_t);
ifs.seekg(0, std::ios::beg);
// read in the data
if (data.size() < numEle)
data.resize(numEle, 0);
real_t tempVal;
for (unsigned int i = 0; i<numEle; i++){
ifs.read ((char *)&tempVal, sizeof(real_t));
data[i] = tempVal;
}
//thrust::copy(tempVec.begin(), tempVec.end(), data.begin());
ifs.close();
return numEle;
}
void ParseFloatOpt(const std::string stringOpt, Cpu::real_vector &optVec){
std::vector<std::string> tempArgs;
std::vector<std::string> tempArgs2;
std::vector<real_t> tmpresult;
if (stringOpt.size()==0){
optVec.clear();
return;
}
boost::split(tempArgs, stringOpt, boost::is_any_of("_"));
for (int i =0 ; i<tempArgs.size(); i++){
boost::split(tempArgs2, tempArgs[i], boost::is_any_of("*"));
if (tempArgs2.size() == 2){
int cnt = boost::lexical_cast<int>(tempArgs2[0]);
for (int j = 0; j < cnt; j++)
tmpresult.push_back(boost::lexical_cast<real_t>(tempArgs2[1]));
}else{
tmpresult.push_back(boost::lexical_cast<real_t>(tempArgs[i]));
}
}
optVec.resize(tmpresult.size(), 0.0);
for (int i=0;i<optVec.size();i++)
optVec[i] = tmpresult[i];
}
void outputWeight(rapidjson::Document *jsonDoc, rapidjson::Document *jsonDoc2, std::unordered_map<std::string, int> *p_dict, std::string outputfilename)
{
if (!jsonDoc->HasMember("layers"))
throw std::runtime_error("Missing section 'layers'");
rapidjson::Value &layersSection = (*jsonDoc)["layers"];
if (! jsonDoc2->HasMember("weights") )
return;
helpers::JsonValue weightsSection;
if (jsonDoc2->HasMember("weights")) {
//if (!(*jsonDoc)["weights"].IsObject())
//throw std::runtime_error("Section 'weights' is not an object");
weightsSection = helpers::JsonValue(&(*jsonDoc2)["weights"]);
}
std::ofstream ofs(outputfilename);
bool found = false;
int beforesize=0;
for (rapidjson::Value::ValueIterator layerChild = layersSection.Begin(); layerChild != layersSection.End(); ++layerChild) {
if (strcmp( (*layerChild)["name"].GetString(), "output" ) != 0 ){
beforesize = (*layerChild)["size"].GetInt();
continue;
}
int esize = beforesize;
printf("found output layer! size : %d\n", esize);
found = true;
if (!weightsSection->HasMember("output"))
throw std::runtime_error("softmax weight doesn't exist");
const rapidjson::Value &weightsChild = (*weightsSection)["output"];
const rapidjson::Value &inputWeightsChild = weightsChild["input"];
const rapidjson::Value &biasWeightsChild = weightsChild["bias"];
Cpu::real_vector weights;
weights.reserve(inputWeightsChild.Size());
for (rapidjson::Value::ConstValueIterator it = inputWeightsChild.Begin(); it != inputWeightsChild.End(); ++it)
weights.push_back(static_cast<real_t>(it->GetDouble()));
// write "WordNum dimension(size)"
ofs << p_dict->size() << " " << esize << std::endl;
if (weights.size() < esize * p_dict->size()){
std::unordered_map<int, std::string> reversed;
for (auto it = p_dict->begin(); it != p_dict->end(); ++it){
reversed[it->second] = it->first;
}
for (int id = 0; id < weights.size() / esize; ++id) {
// writting word
ofs << reversed[id] << " ";
int startId = id * esize;
// writting vector
for (int i = 0; i < esize; ++i)
ofs << weights[startId + i] << " ";
ofs << std::endl;
}
}
else {
for (auto it = p_dict->begin(); it != p_dict->end(); ++it){
std::string word = it->first;
int startId = it->second * esize;
ofs << word << " ";
for (int i = 0; i < esize; ++i)
ofs << weights[startId + i] << " ";
ofs << std::endl;
}
}
}
if(!found)
throw std::runtime_error("softmax layer's information doesn't exist in layer section");
}