int MomentumConn::updateWeights(int arborId){
if(timeBatchIdx != timeBatchPeriod - 1){
return PV_SUCCESS;
}
//Add momentum right before updateWeights
for(int kArbor = 0; kArbor < this->numberOfAxonalArborLists(); kArbor++){
applyMomentum(arborId);
}
//Saved to prevweights
assert(prev_dwDataStart);
std::memcpy(*prev_dwDataStart, *get_dwDataStart(),
sizeof(pvwdata_t) *
numberOfAxonalArborLists() *
nxp * nyp * nfp *
getNumDataPatches());
// add dw to w
for(int kArbor = 0; kArbor < this->numberOfAxonalArborLists(); kArbor++){
pvwdata_t * w_data_start = get_wDataStart(kArbor);
for( long int k=0; k<patchStartIndex(getNumDataPatches()); k++ ) {
w_data_start[k] += get_dwDataStart(kArbor)[k];
}
}
return PV_BREAK;
}
int LCALIFLateralKernelConn::checkpointWrite(const char * cpDir) {
int status = HyPerConn::checkpointWrite(cpDir);
char filename[PV_PATH_MAX];
int chars_needed = snprintf(filename, PV_PATH_MAX, "%s/%s_integratedSpikeCount.pvp", cpDir, name);
if (chars_needed >= PV_PATH_MAX) {
fprintf(stderr, "LCALIFLateralKernelConn::checkpointWrite error. Path \"%s/%s_integratedSpikeCount.pvp\" is too long.\n", cpDir, name);
abort();
}
int status2 = HyPerLayer::writeBufferFile(filename, parent->icCommunicator(), parent->simulationTime(), &integratedSpikeCount, 1/*numbands*/, false/*extended*/, pre->getLayerLoc());
if (status2!=PV_SUCCESS) status = status2;
chars_needed = snprintf(filename, PV_PATH_MAX, "%s/%s_dW.pvp", cpDir, name);
assert(chars_needed < PV_PATH_MAX);
status2 = HyPerConn::writeWeights(NULL, get_dwDataStart(), getNumDataPatches(), filename, parent->simulationTime(), writeCompressedCheckpoints, true);
if (status2!=PV_SUCCESS) status = status2;
return status;
}
//Connections update first
int GradientCheckConn::updateState(double time, double dt){
int status = PV_SUCCESS;
int weightIdx = parent->getCurrentStep() - parent->getInitialStep() - 2;
std::cout << "weightIdx " << weightIdx << "\n";
int numPatch = nxp * nyp * nfp;
int numData = getNumDataPatches();
int arborIdx = weightIdx / (numPatch * numData);
int dataIdx = (weightIdx / numPatch) % numData;
int patchIdx = weightIdx % numPatch;
if(firstRun){
initialize_dW(0);
firstRun = false;
return PV_SUCCESS;
}
//Grab cost from previous timestep
if(secondRun){
//First run does regular updateState to calculate dw buffer
for(int arborId=0;arborId<numberOfAxonalArborLists();arborId++) {
status = calc_dW(); // Calculate changes in weights
if (status==PV_BREAK) { break; }
assert(status == PV_SUCCESS);
}
//for (int arborID = 0; arborID < numberOfAxonalArborLists(); arborID++) {
// if(sharedWeights){
// status = reduceKernels(arborID); // combine partial changes in each column
// if (status == PV_BREAK) {
// break;
// }
// assert(status == PV_SUCCESS);
// }
//}
//No update weights
origCost = getCost();
secondRun = false;
}
//Does not update after first run
//Check if we are in bounds for non-shared weights
if(!sharedWeights){
PVPatch* weights = getWeights(dataIdx, arborIdx);
//Calculate x and y of patchIdx and compare it to offset
int xPatchIdx = kxPos(patchIdx, nxp, nyp, nfp);
int yPatchIdx = kyPos(patchIdx, nxp, nyp, nfp);
int xOffsetIdx = kxPos(weights->offset, nxp, nyp, nfp);
int yOffsetIdx = kyPos(weights->offset, nxp, nyp, nfp);
//If index is oob, skip
if(xPatchIdx < xOffsetIdx || xPatchIdx >= xOffsetIdx + weights->nx ||
yPatchIdx < yOffsetIdx || yPatchIdx >= yOffsetIdx + weights->ny){
return PV_SUCCESS;
}
}
//Calculate difference in numerical method and backprop method
if(prevIdx != -1){
currCost = getCost();
//Check for accuracy
float numGradient = (currCost - origCost)/epsilon;
float backpropGradient = get_dwDataStart()[0][prevIdx] / dWMax;
std::cout << "Numerical gradient: " << numGradient << " Backprop gradient: " << backpropGradient << "\n";
//if(fabs(numGradient + backpropGradient) >= .1){
// std::cout << "Numerical gradient: " << numGradient << " Backprop gradient: " << backpropGradient << "\n";
// exit(-1);
//}
}
//Restore weight
if(prevIdx != -1){
std::cout << "Restoring weight " << prevIdx << " to " << prevWeightVal << "\n";
get_wDataStart()[0][prevIdx] = prevWeightVal;
}
//Set next weight if not the end
if(weightIdx < numberOfAxonalArborLists() * numData * numPatch){
prevWeightVal = get_wDataStart()[0][weightIdx];
prevIdx = weightIdx;
get_wDataStart()[0][weightIdx] += epsilon;
std::cout << "Setting weight " << weightIdx << " to " << prevWeightVal + epsilon << "\n";
}
else{
std::cout << "END\n";
}
return status;
}
int MapReduceKernelConn::reduceKernels(const int arborID) {
int status = HyPerConn::reduceKernels(arborID);
int rootproc = 0;
InterColComm *icComm = parent->icCommunicator();
const int numPatches = getNumDataPatches();
const size_t patchSize = nxp * nyp * nfp * sizeof(pvdata_t);
const size_t localSize = numPatches * patchSize;
const size_t arborSize = localSize * this->numberOfAxonalArborLists();
if (icComm->commRank() == rootproc) {
// write dW for this instantiation of PetaVision to disk
status = HyPerConn::writeWeights(NULL, this->get_dwDataStart(),
getNumDataPatches(), dWeightsList[dWeightFileIndex],
parent->simulationTime(), /*writeCompressedWeights*/false, /*last*/
false);
if (status != PV_SUCCESS) {
fprintf(stderr,
"MapReduceKernelConn::reduceKernels::HyPerConn::writeWeights: problem writing to file %s, "
"SHUTTING DOWN\n", dWeightsList[dWeightFileIndex]);
exit(EXIT_FAILURE);
} // status
// use dWeightsList to read in the weights written by other PetaVision instantiations
double dW_time;
double simulation_time = parent->simulationTime();
int filetype, datatype;
int numParams = NUM_BIN_PARAMS + NUM_WGT_EXTRA_PARAMS;
int params[NUM_BIN_PARAMS + NUM_WGT_EXTRA_PARAMS];
const PVLayerLoc *preLoc = this->preSynapticLayer()->getLayerLoc();
int file_count = 0;
for (file_count = 0; file_count < num_dWeightFiles; file_count++) {
if (file_count == dWeightFileIndex) {
continue;
}
int num_attempts = 0;
const int MAX_ATTEMPTS = 5;
dW_time = 0;
while (dW_time < simulation_time && num_attempts <= MAX_ATTEMPTS) {
pvp_read_header(dWeightsList[file_count], icComm, &dW_time,
&filetype, &datatype, params, &numParams);
num_attempts++;
} // while
if (num_attempts > MAX_ATTEMPTS) {
fprintf(stderr,
"PV::MapReduceKernelConn::reduceKernels: problem reading arbor file %s, SHUTTING DOWN\n",
dWeightsList[file_count]);
status = EXIT_FAILURE;
exit(EXIT_FAILURE);
} // num_attempts > MAX_ATTEMPTS
int status = PV::readWeights(NULL, get_dwDataStart(),
this->numberOfAxonalArborLists(), this->getNumDataPatches(), nxp, nyp, nfp,
dWeightsList[file_count], icComm, &dW_time, preLoc);
if (status != PV_SUCCESS) {
fprintf(stderr,
"MapReduceKernelConn::reduceKernels::PV::readWeights: problem reading file %s, "
"SHUTTING DOWN\n", dWeightsList[file_count]);
exit(EXIT_FAILURE);
} // status
} // file_count < numWeightFiles
// average dW from map-reduce
pvwdata_t * dW_data = this->get_dwDataStart(0);
for (int i_dW = 0; i_dW < arborSize; i_dW++) {
dW_data[i_dW] /= num_dWeightFiles;
}
} // rootproc
// broadcast map-reduced dWeights to all non-root processes
MPI_Comm mpi_comm = icComm->communicator();
#ifdef PV_USE_MPI
MPI_Bcast(this->get_wDataStart(0), arborSize, MPI_FLOAT, rootproc, mpi_comm);
#endif
return PV_BREAK;
}
