예제 #1
0
int ReadRealData(const std::string dataPath, Cpu::real_vector &data)
{
    // 
    std::ifstream ifs(dataPath.c_str(), std::ifstream::binary | std::ifstream::in);
    if (!ifs.good())
	throw std::runtime_error(std::string("Fail to open ")+dataPath);
    
    // get the number of we data
    std::streampos numEleS, numEleE;
    long int numEle;
    numEleS = ifs.tellg();
    ifs.seekg(0, std::ios::end);
    numEleE = ifs.tellg();
    numEle  = (numEleE-numEleS)/sizeof(real_t);
    ifs.seekg(0, std::ios::beg);
    
    // read in the data
    if (data.size() < numEle)
	data.resize(numEle, 0);
    
    real_t tempVal;
    for (unsigned int i = 0; i<numEle; i++){
	ifs.read ((char *)&tempVal, sizeof(real_t));
	data[i] = tempVal;
    }
    //thrust::copy(tempVec.begin(), tempVec.end(), data.begin());
    ifs.close();
    return numEle;
}
예제 #2
0
void ParseFloatOpt(const std::string stringOpt, Cpu::real_vector &optVec){
    std::vector<std::string> tempArgs;
    std::vector<std::string> tempArgs2;
    std::vector<real_t> tmpresult;

    if (stringOpt.size()==0){
	optVec.clear();
	return;
    }
    
    boost::split(tempArgs, stringOpt, boost::is_any_of("_"));
    for (int i =0 ; i<tempArgs.size(); i++){
	boost::split(tempArgs2, tempArgs[i], boost::is_any_of("*"));
	if (tempArgs2.size() == 2){
	    int cnt = boost::lexical_cast<int>(tempArgs2[0]);
	    for (int j = 0; j < cnt; j++)
		tmpresult.push_back(boost::lexical_cast<real_t>(tempArgs2[1]));
	}else{
	    tmpresult.push_back(boost::lexical_cast<real_t>(tempArgs[i]));
	}
    }
    optVec.resize(tmpresult.size(), 0.0);
    for (int i=0;i<optVec.size();i++)
	optVec[i] = tmpresult[i];
}
예제 #3
0
void outputWeight(rapidjson::Document *jsonDoc, rapidjson::Document *jsonDoc2, std::unordered_map<std::string, int> *p_dict, std::string outputfilename)
{

    if (!jsonDoc->HasMember("layers"))
        throw std::runtime_error("Missing section 'layers'");
    rapidjson::Value &layersSection  = (*jsonDoc)["layers"];

    if (! jsonDoc2->HasMember("weights") )
        return;
    helpers::JsonValue weightsSection;
    if (jsonDoc2->HasMember("weights")) {
        //if (!(*jsonDoc)["weights"].IsObject())
            //throw std::runtime_error("Section 'weights' is not an object");

        weightsSection = helpers::JsonValue(&(*jsonDoc2)["weights"]);
    }
    std::ofstream ofs(outputfilename);
    bool found = false;
    int beforesize=0;
    for (rapidjson::Value::ValueIterator layerChild = layersSection.Begin(); layerChild != layersSection.End(); ++layerChild) {
        if (strcmp( (*layerChild)["name"].GetString(), "output" ) != 0 ){
            beforesize = (*layerChild)["size"].GetInt();
            continue;
        }
        int esize = beforesize;
        printf("found output layer! size : %d\n", esize);
        found = true;
        if (!weightsSection->HasMember("output"))
            throw std::runtime_error("softmax weight doesn't exist");
        const rapidjson::Value &weightsChild = (*weightsSection)["output"];
        const rapidjson::Value &inputWeightsChild = weightsChild["input"];
        const rapidjson::Value &biasWeightsChild  = weightsChild["bias"];

        Cpu::real_vector weights;
        weights.reserve(inputWeightsChild.Size());
        for (rapidjson::Value::ConstValueIterator it = inputWeightsChild.Begin(); it != inputWeightsChild.End(); ++it)
            weights.push_back(static_cast<real_t>(it->GetDouble()));

        // write "WordNum dimension(size)"
        ofs << p_dict->size() << " " << esize << std::endl;
        if (weights.size() < esize * p_dict->size()){
            std::unordered_map<int, std::string> reversed;
            for (auto it = p_dict->begin(); it != p_dict->end(); ++it){
                reversed[it->second] = it->first;
            }
            for (int id = 0; id < weights.size() / esize; ++id) {
                // writting word
                ofs << reversed[id] << " ";
                int startId = id * esize;
                // writting vector
                for (int i = 0; i < esize; ++i)
                    ofs << weights[startId + i] << " ";
                ofs << std::endl;
            }
        }
        else {
           for (auto it = p_dict->begin(); it != p_dict->end(); ++it){
               std::string word = it->first;
               int startId = it->second * esize;
               ofs << word << " ";
               for (int i = 0; i < esize; ++i)
                   ofs << weights[startId + i] << " ";
               ofs << std::endl;
            }
         }
    }
    if(!found)
        throw std::runtime_error("softmax layer's information doesn't exist in layer section");
}