int MomentumConn::updateWeights(int arborId){ if(timeBatchIdx != timeBatchPeriod - 1){ return PV_SUCCESS; } //Add momentum right before updateWeights for(int kArbor = 0; kArbor < this->numberOfAxonalArborLists(); kArbor++){ applyMomentum(arborId); } //Saved to prevweights assert(prev_dwDataStart); std::memcpy(*prev_dwDataStart, *get_dwDataStart(), sizeof(pvwdata_t) * numberOfAxonalArborLists() * nxp * nyp * nfp * getNumDataPatches()); // add dw to w for(int kArbor = 0; kArbor < this->numberOfAxonalArborLists(); kArbor++){ pvwdata_t * w_data_start = get_wDataStart(kArbor); for( long int k=0; k<patchStartIndex(getNumDataPatches()); k++ ) { w_data_start[k] += get_dwDataStart(kArbor)[k]; } } return PV_BREAK; }
int LCALIFLateralKernelConn::checkpointWrite(const char * cpDir) { int status = HyPerConn::checkpointWrite(cpDir); char filename[PV_PATH_MAX]; int chars_needed = snprintf(filename, PV_PATH_MAX, "%s/%s_integratedSpikeCount.pvp", cpDir, name); if (chars_needed >= PV_PATH_MAX) { fprintf(stderr, "LCALIFLateralKernelConn::checkpointWrite error. Path \"%s/%s_integratedSpikeCount.pvp\" is too long.\n", cpDir, name); abort(); } int status2 = HyPerLayer::writeBufferFile(filename, parent->icCommunicator(), parent->simulationTime(), &integratedSpikeCount, 1/*numbands*/, false/*extended*/, pre->getLayerLoc()); if (status2!=PV_SUCCESS) status = status2; chars_needed = snprintf(filename, PV_PATH_MAX, "%s/%s_dW.pvp", cpDir, name); assert(chars_needed < PV_PATH_MAX); status2 = HyPerConn::writeWeights(NULL, get_dwDataStart(), getNumDataPatches(), filename, parent->simulationTime(), writeCompressedCheckpoints, true); if (status2!=PV_SUCCESS) status = status2; return status; }
//Connections update first int GradientCheckConn::updateState(double time, double dt){ int status = PV_SUCCESS; int weightIdx = parent->getCurrentStep() - parent->getInitialStep() - 2; std::cout << "weightIdx " << weightIdx << "\n"; int numPatch = nxp * nyp * nfp; int numData = getNumDataPatches(); int arborIdx = weightIdx / (numPatch * numData); int dataIdx = (weightIdx / numPatch) % numData; int patchIdx = weightIdx % numPatch; if(firstRun){ initialize_dW(0); firstRun = false; return PV_SUCCESS; } //Grab cost from previous timestep if(secondRun){ //First run does regular updateState to calculate dw buffer for(int arborId=0;arborId<numberOfAxonalArborLists();arborId++) { status = calc_dW(); // Calculate changes in weights if (status==PV_BREAK) { break; } assert(status == PV_SUCCESS); } //for (int arborID = 0; arborID < numberOfAxonalArborLists(); arborID++) { // if(sharedWeights){ // status = reduceKernels(arborID); // combine partial changes in each column // if (status == PV_BREAK) { // break; // } // assert(status == PV_SUCCESS); // } //} //No update weights origCost = getCost(); secondRun = false; } //Does not update after first run //Check if we are in bounds for non-shared weights if(!sharedWeights){ PVPatch* weights = getWeights(dataIdx, arborIdx); //Calculate x and y of patchIdx and compare it to offset int xPatchIdx = kxPos(patchIdx, nxp, nyp, nfp); int yPatchIdx = kyPos(patchIdx, nxp, nyp, nfp); int xOffsetIdx = kxPos(weights->offset, nxp, nyp, nfp); int yOffsetIdx = kyPos(weights->offset, nxp, nyp, nfp); //If index is oob, skip if(xPatchIdx < xOffsetIdx || xPatchIdx >= xOffsetIdx + weights->nx || yPatchIdx < yOffsetIdx || yPatchIdx >= yOffsetIdx + weights->ny){ return PV_SUCCESS; } } //Calculate difference in numerical method and backprop method if(prevIdx != -1){ currCost = getCost(); //Check for accuracy float numGradient = (currCost - origCost)/epsilon; float backpropGradient = get_dwDataStart()[0][prevIdx] / dWMax; std::cout << "Numerical gradient: " << numGradient << " Backprop gradient: " << backpropGradient << "\n"; //if(fabs(numGradient + backpropGradient) >= .1){ // std::cout << "Numerical gradient: " << numGradient << " Backprop gradient: " << backpropGradient << "\n"; // exit(-1); //} } //Restore weight if(prevIdx != -1){ std::cout << "Restoring weight " << prevIdx << " to " << prevWeightVal << "\n"; get_wDataStart()[0][prevIdx] = prevWeightVal; } //Set next weight if not the end if(weightIdx < numberOfAxonalArborLists() * numData * numPatch){ prevWeightVal = get_wDataStart()[0][weightIdx]; prevIdx = weightIdx; get_wDataStart()[0][weightIdx] += epsilon; std::cout << "Setting weight " << weightIdx << " to " << prevWeightVal + epsilon << "\n"; } else{ std::cout << "END\n"; } return status; }
int MapReduceKernelConn::reduceKernels(const int arborID) { int status = HyPerConn::reduceKernels(arborID); int rootproc = 0; InterColComm *icComm = parent->icCommunicator(); const int numPatches = getNumDataPatches(); const size_t patchSize = nxp * nyp * nfp * sizeof(pvdata_t); const size_t localSize = numPatches * patchSize; const size_t arborSize = localSize * this->numberOfAxonalArborLists(); if (icComm->commRank() == rootproc) { // write dW for this instantiation of PetaVision to disk status = HyPerConn::writeWeights(NULL, this->get_dwDataStart(), getNumDataPatches(), dWeightsList[dWeightFileIndex], parent->simulationTime(), /*writeCompressedWeights*/false, /*last*/ false); if (status != PV_SUCCESS) { fprintf(stderr, "MapReduceKernelConn::reduceKernels::HyPerConn::writeWeights: problem writing to file %s, " "SHUTTING DOWN\n", dWeightsList[dWeightFileIndex]); exit(EXIT_FAILURE); } // status // use dWeightsList to read in the weights written by other PetaVision instantiations double dW_time; double simulation_time = parent->simulationTime(); int filetype, datatype; int numParams = NUM_BIN_PARAMS + NUM_WGT_EXTRA_PARAMS; int params[NUM_BIN_PARAMS + NUM_WGT_EXTRA_PARAMS]; const PVLayerLoc *preLoc = this->preSynapticLayer()->getLayerLoc(); int file_count = 0; for (file_count = 0; file_count < num_dWeightFiles; file_count++) { if (file_count == dWeightFileIndex) { continue; } int num_attempts = 0; const int MAX_ATTEMPTS = 5; dW_time = 0; while (dW_time < simulation_time && num_attempts <= MAX_ATTEMPTS) { pvp_read_header(dWeightsList[file_count], icComm, &dW_time, &filetype, &datatype, params, &numParams); num_attempts++; } // while if (num_attempts > MAX_ATTEMPTS) { fprintf(stderr, "PV::MapReduceKernelConn::reduceKernels: problem reading arbor file %s, SHUTTING DOWN\n", dWeightsList[file_count]); status = EXIT_FAILURE; exit(EXIT_FAILURE); } // num_attempts > MAX_ATTEMPTS int status = PV::readWeights(NULL, get_dwDataStart(), this->numberOfAxonalArborLists(), this->getNumDataPatches(), nxp, nyp, nfp, dWeightsList[file_count], icComm, &dW_time, preLoc); if (status != PV_SUCCESS) { fprintf(stderr, "MapReduceKernelConn::reduceKernels::PV::readWeights: problem reading file %s, " "SHUTTING DOWN\n", dWeightsList[file_count]); exit(EXIT_FAILURE); } // status } // file_count < numWeightFiles // average dW from map-reduce pvwdata_t * dW_data = this->get_dwDataStart(0); for (int i_dW = 0; i_dW < arborSize; i_dW++) { dW_data[i_dW] /= num_dWeightFiles; } } // rootproc // broadcast map-reduced dWeights to all non-root processes MPI_Comm mpi_comm = icComm->communicator(); #ifdef PV_USE_MPI MPI_Bcast(this->get_wDataStart(0), arborSize, MPI_FLOAT, rootproc, mpi_comm); #endif return PV_BREAK; }