int main(int argc, const char *const argv[]) { int err, ret = EXIT_FAILURE; FILE *f, *infile = stdin; double start, end; unsigned int nnodes; struct node *nodes; if (argc != 3) usage(); f = fopen(argv[1], "r"); if (!f) { perror("Error opening data file"); return EXIT_FAILURE; } if (strcmp(argv[2], "-") != 0) { infile = fopen(argv[2], "r"); if (!infile) { perror("Error opening data file"); fclose(f); return EXIT_FAILURE; } } start = get_current_seconds(); err = load_map(f, &nodes, &nnodes); if (err) goto out_map; end = get_current_seconds(); fclose(f); if (err) return EXIT_FAILURE; printf("Loaded %d nodes in %f seconds\n", nnodes, end - start); printf("Using %d MB\n", peak_memory_usage()); err = input_and_search(infile, nodes, nnodes); if (err) goto out; printf("Peak memory usage %d MB\n", peak_memory_usage()); ret = EXIT_SUCCESS; out: free_map(nodes, nnodes); out_map: if (infile != stderr) fclose(infile); return ret; }
bool MultiShearCatalog::getPixels(const Bounds& bounds) { // The pixlist object takes up a lot of memory, so at the start // of this function, I clear it out, along with psflist, etc. // Then we loop over sections of the coaddCat and only load the information // for each single-epoch observation that actually falls within the // bounds for the section we are working on. const int nPix = _pix_list.size(); dbg<<"Start getPixels: memory_usage = "<<memory_usage()<<std::endl; for (int i=0;i<nPix;++i) { _pix_list[i].clear(); _psf_list[i].clear(); } dbg<<"After clear: memory_usage = "<<memory_usage()<<std::endl; bool des_qa = _params.read("des_qa",false); try { dbg<<"Start GetPixels for b = "<<bounds<<std::endl; memory_usage(dbgout); // Loop over the files and read pixel lists for each object. // The Transformation and FittedPSF constructors get the name // information from the parameter file, so we use that to set the // names of each component image here. const int nfiles = _image_file_list.size(); for (int ifile=0; ifile<nfiles; ++ifile) { #ifdef ONLY_N_IMAGES if (ifile >= ONLY_N_IMAGES) break; #endif dbg<<"ifile = "<<ifile<<std::endl; // Get the file names Assert(ifile < int(_image_file_list.size())); Assert(ifile < int(_fitpsf_file_list.size())); std::string image_file = _image_file_list[ifile]; std::string fitpsf_file = _fitpsf_file_list[ifile]; dbg<<"Reading image file: "<<image_file<<"\n"; // Set the appropriate parameters _params["image_file"] = image_file; SetRoot(_params,image_file); _params["fitpsf_file"] = fitpsf_file; if (_shear_file_list.size() > 0) { Assert(ifile < int(_shear_file_list.size())); std::string shear_file = _shear_file_list[ifile]; _params["shear_file"] = shear_file; } if (_skymap_file_list.size() > 0) { Assert(ifile < int(_skymap_file_list.size())); std::string skymap_file = _skymap_file_list[ifile]; _params["skymap_file"] = skymap_file; } // Load the pixels dbg<<"Before load pixels for file "<<ifile<< ": memory_usage = "<<memory_usage()<<std::endl; if (!getImagePixelLists(ifile,bounds)) { for (int i=0;i<nPix;++i) { _pix_list[i].clear(); _psf_list[i].clear(); } PixelList::reclaimMemory(); return false; } dbg<<"After load pixels for file "<<ifile<< ": memory_usage = "<<memory_usage()<<std::endl; } } catch (std::bad_alloc) { dbg<<"Caught bad_alloc\n"; double mem = memory_usage(dbgout); double peak_mem = peak_memory_usage(); dbg<<"memory usage = "<<mem<<" MB\n"; dbg<<"peak memory usage = "<<peak_mem<<" MB\n"; if (des_qa) std::cerr<<"STATUS5BEG "; std::cerr << "Memory exhausted in MultShearCatalog.\n" << "Memory Usage in MultiShearCatalog = " << calculateMemoryFootprint()<<" MB \n" << "Actual Virtual Memory Usage = " << mem<<" MB \n" << "Try reducing multishear_section_size or " << "reducing mam_vmem\n" << "(Current values = " << _params["multishear_section_size"] << " , "<<_params["max_vmem"]<<")"; if (des_qa) std::cerr<<" STATUS5END"; std::cerr<<std::endl; dbg << "Memory exhausted in MultShearCatalog.\n" << "Memory Usage in MultiShearCatalog = " << calculateMemoryFootprint()<<" MB \n" << "Actual Virtual Memory Usage = " << mem<<" MB \n" << "Try reducing multishear_section_size or " << "reducing mam_vmem\n" << "(Current values = " << _params["multishear_section_size"] << " , "<<_params["max_vmem"]<<")" << std::endl; exit(1); } dbg <<"Done getPixels\n"; dbg << "Memory Usage in MultiShearCatalog = " << calculateMemoryFootprint()<<" MB \n"; double mem = memory_usage(dbgout); double peak_mem = peak_memory_usage(); double max_mem = double(_params["max_vmem"])*1024.; _params["peak_mem"] = peak_mem; // Keep track... dbg<<"Actual memory usage = "<<mem<<" MB\n"; dbg<<"Peak memory usage = "<<peak_mem<<" MB\n"; dbg<<"Max allowed memory usage = "<<max_mem<<" MB\n"; return true; }