int ReadRealData(const std::string dataPath, Cpu::real_vector &data) { // std::ifstream ifs(dataPath.c_str(), std::ifstream::binary | std::ifstream::in); if (!ifs.good()) throw std::runtime_error(std::string("Fail to open ")+dataPath); // get the number of we data std::streampos numEleS, numEleE; long int numEle; numEleS = ifs.tellg(); ifs.seekg(0, std::ios::end); numEleE = ifs.tellg(); numEle = (numEleE-numEleS)/sizeof(real_t); ifs.seekg(0, std::ios::beg); // read in the data if (data.size() < numEle) data.resize(numEle, 0); real_t tempVal; for (unsigned int i = 0; i<numEle; i++){ ifs.read ((char *)&tempVal, sizeof(real_t)); data[i] = tempVal; } //thrust::copy(tempVec.begin(), tempVec.end(), data.begin()); ifs.close(); return numEle; }
void ParseFloatOpt(const std::string stringOpt, Cpu::real_vector &optVec){ std::vector<std::string> tempArgs; std::vector<std::string> tempArgs2; std::vector<real_t> tmpresult; if (stringOpt.size()==0){ optVec.clear(); return; } boost::split(tempArgs, stringOpt, boost::is_any_of("_")); for (int i =0 ; i<tempArgs.size(); i++){ boost::split(tempArgs2, tempArgs[i], boost::is_any_of("*")); if (tempArgs2.size() == 2){ int cnt = boost::lexical_cast<int>(tempArgs2[0]); for (int j = 0; j < cnt; j++) tmpresult.push_back(boost::lexical_cast<real_t>(tempArgs2[1])); }else{ tmpresult.push_back(boost::lexical_cast<real_t>(tempArgs[i])); } } optVec.resize(tmpresult.size(), 0.0); for (int i=0;i<optVec.size();i++) optVec[i] = tmpresult[i]; }
void outputWeight(rapidjson::Document *jsonDoc, rapidjson::Document *jsonDoc2, std::unordered_map<std::string, int> *p_dict, std::string outputfilename) { if (!jsonDoc->HasMember("layers")) throw std::runtime_error("Missing section 'layers'"); rapidjson::Value &layersSection = (*jsonDoc)["layers"]; if (! jsonDoc2->HasMember("weights") ) return; helpers::JsonValue weightsSection; if (jsonDoc2->HasMember("weights")) { //if (!(*jsonDoc)["weights"].IsObject()) //throw std::runtime_error("Section 'weights' is not an object"); weightsSection = helpers::JsonValue(&(*jsonDoc2)["weights"]); } std::ofstream ofs(outputfilename); bool found = false; int beforesize=0; for (rapidjson::Value::ValueIterator layerChild = layersSection.Begin(); layerChild != layersSection.End(); ++layerChild) { if (strcmp( (*layerChild)["name"].GetString(), "output" ) != 0 ){ beforesize = (*layerChild)["size"].GetInt(); continue; } int esize = beforesize; printf("found output layer! size : %d\n", esize); found = true; if (!weightsSection->HasMember("output")) throw std::runtime_error("softmax weight doesn't exist"); const rapidjson::Value &weightsChild = (*weightsSection)["output"]; const rapidjson::Value &inputWeightsChild = weightsChild["input"]; const rapidjson::Value &biasWeightsChild = weightsChild["bias"]; Cpu::real_vector weights; weights.reserve(inputWeightsChild.Size()); for (rapidjson::Value::ConstValueIterator it = inputWeightsChild.Begin(); it != inputWeightsChild.End(); ++it) weights.push_back(static_cast<real_t>(it->GetDouble())); // write "WordNum dimension(size)" ofs << p_dict->size() << " " << esize << std::endl; if (weights.size() < esize * p_dict->size()){ std::unordered_map<int, std::string> reversed; for (auto it = p_dict->begin(); it != p_dict->end(); ++it){ reversed[it->second] = it->first; } for (int id = 0; id < weights.size() / esize; ++id) { // writting word ofs << reversed[id] << " "; int startId = id * esize; // writting vector for (int i = 0; i < esize; ++i) ofs << weights[startId + i] << " "; ofs << std::endl; } } else { for (auto it = p_dict->begin(); it != p_dict->end(); ++it){ std::string word = it->first; int startId = it->second * esize; ofs << word << " "; for (int i = 0; i < esize; ++i) ofs << weights[startId + i] << " "; ofs << std::endl; } } } if(!found) throw std::runtime_error("softmax layer's information doesn't exist in layer section"); }