void DataSetList::PrintList(DataListType const& dlist) { for (DataListType::const_iterator ds = dlist.begin(); ds != dlist.end(); ++ds) { DataSet const& dset = static_cast<DataSet const&>( *(*ds) ); mprintf("\t%s \"%s\" (%s%s), size is %zu", dset.Meta().PrintName().c_str(), dset.legend(), DataArray[dset.Type()].Description, dset.Meta().ScalarDescription().c_str(), dset.Size()); dset.Info(); mprintf("\n"); } }
// DataSetList::Clear() void DataSetList::Clear() { DataListType setsToErase; DataListType setsToKeep; setsToKeep.reserve( RefList_.size() + TopList_.size() ); for (DataListType::const_iterator ds = DataList_.begin(); ds != DataList_.end(); ++ds) if ( (*ds)->Type() == DataSet::REF_FRAME || (*ds)->Type() == DataSet::TOPOLOGY ) setsToKeep.push_back( *ds ); else setsToErase.push_back( *ds ); if (!hasCopies_) for (DataListType::iterator ds = setsToErase.begin(); ds != setsToErase.end(); ++ds) delete *ds; DataList_ = setsToKeep; }
/** Given an array of already set up data sets (from e.g. input data files) * and optional X values, add the sets to this list if they do not exist * or append any existing sets. */ int DataSetList::AddOrAppendSets(std::string const& XlabelIn, Darray const& Xvals, DataListType const& Sets) { if (debug_ > 0) mprintf("DEBUG: Calling AddOrAppendSets for %zu sets, %zu X values, Xlabel= %s.\n", Sets.size(), Xvals.size(), XlabelIn.c_str()); if (Sets.empty()) return 0; // No error for now. // If no X label assume 'Frame' for backwards compat. std::string Xlabel; if (XlabelIn.empty()) Xlabel.assign("Frame"); else Xlabel = XlabelIn; Dimension Xdim; // First determine if X values increase monotonically with a regular step bool isMonotonic = true; double xstep = 1.0; if (Xvals.size() > 1) { xstep = (Xvals.back() - Xvals.front()) / (double)(Xvals.size() - 1); for (Darray::const_iterator X = Xvals.begin()+2; X != Xvals.end(); ++X) if ((*X - *(X-1)) - xstep > Constants::SMALL) { isMonotonic = false; break; } // Set dim even for non-monotonic sets so Label is correct. FIXME is this ok? Xdim = Dimension( Xvals.front(), xstep, Xlabel ); } else // No X values. set generic X dim. Xdim = Dimension(1.0, 1.0, Xlabel); if (debug_ > 0) { mprintf("DEBUG: xstep %g xmin %g\n", Xdim.Step(), Xdim.Min()); if (isMonotonic) mprintf("DEBUG: Xdim is monotonic.\n"); } for (const_iterator ds = Sets.begin(); ds != Sets.end(); ++ds) { if (*ds == 0) continue; // TODO: Warn about blanks? if (debug_ > 0) mprintf("DEBUG: AddOrAppend set '%s'", (*ds)->legend()); if (isMonotonic) (*ds)->SetDim(Dimension::X, Xdim); DataSet* existingSet = CheckForSet( (*ds)->Meta() ); if (existingSet == 0) { // New set. If set is scalar 1D but X values are not monotonic, // convert to XY mesh if necessary. if ( !isMonotonic && (*ds)->Group() == DataSet::SCALAR_1D && (*ds)->Type() != DataSet::XYMESH ) { DataSet* xyptr = AddSet( DataSet::XYMESH, (*ds)->Meta() ); if (xyptr == 0) { mprinterr("Error: Could not convert set %s to XY mesh.\n", (*ds)->legend()); continue; // TODO exit instead? } if ( (*ds)->Size() != Xvals.size() ) { // Sanity check mprinterr("Error: # of X values does not match set %s size.\n", (*ds)->legend()); continue; } DataSet_1D const& set = static_cast<DataSet_1D const&>( *(*ds) ); DataSet_Mesh& xy = static_cast<DataSet_Mesh&>( *xyptr ); for (unsigned int i = 0; i != set.Size(); i++) xy.AddXY( Xvals[i], set.Dval(i) ); xy.SetDim(Dimension::X, Xdim); if (debug_ > 0) mprintf(", New set, converted to XY-MESH\n"); // Free memory since set has now been converted. delete *ds; } else { // No conversion. Just add. (*ds)->SetDim(Dimension::X, Xdim); AddSet( *ds ); if (debug_ > 0) mprintf(", New set\n"); } } else { if (debug_ > 0) mprintf(", appending to existing set\n"); // Set exists. Try to append this set to existing set. bool canAppend = true; if ( (*ds)->Group() == DataSet::GENERIC ) { // For GENERIC sets, base Type must match. // TODO: Should this logic be in DataSets? if ( (*ds)->Type() != existingSet->Type() ) canAppend = false; } else { // For non-GENERIC sets, Group must match if ( (*ds)->Group() != existingSet->Group() ) canAppend = false; } if (!canAppend) mprinterr("Error: Cannot append set of type %s to set of type %s\n", DataArray[(*ds)->Type()].Description, DataArray[existingSet->Type()].Description); // If cannot append or attempt to append fails, rename and add as new set. if (!canAppend || existingSet->Append( *ds )) { // TODO Dimension check? if (canAppend) mprintf("Warning: Append currently not supported for type %s\n", DataArray[existingSet->Type()].Description); MetaData md = (*ds)->Meta(); md.SetName( GenerateDefaultName("X") ); mprintf("Warning: Renaming %s to %s\n", (*ds)->Meta().PrintName().c_str(), md.PrintName().c_str()); (*ds)->SetMeta( md ); AddSet( *ds ); } else // Set was appended to existing set; memory can be freed. delete *ds; } } // END loop over input sets return 0; }
/** Synchronize timeseries data from child ranks to master. */ int DataSetList::SynchronizeData(Parallel::Comm const& commIn) { if (commIn.Size() < 2) return 0; // Ensure that the number of sets that require sync is same on each rank. // FIXME: Make sure this allgather does not end up taking too much time. // Should it be debug only? std::vector<int> size_on_rank; size_on_rank.reserve( DataList_.size() ); DataListType SetsToSync; for (DataListType::iterator ds = DataList_.begin(); ds != DataList_.end(); ++ds) if ( (*ds)->NeedsSync() ) { SetsToSync.push_back( *ds ); size_on_rank.push_back( (*ds)->Size() ); } // DEBUG //for (int rank = 0; rank != commIn.Size(); rank++) { // if (rank == commIn.Rank()) // for (DataListType::const_iterator ds = SetsToSync.begin(); ds != SetsToSync.end(); ++ds) // rprintf("SET '%s'\n", (*ds)->legend()); // commIn.Barrier(); //} // DEBUG END std::vector<int> n_on_rank( commIn.Size(), 0 ); int nSets = (int)SetsToSync.size(); commIn.AllGather( &nSets, 1, MPI_INT, &n_on_rank[0] ); for (int rank = 1; rank < commIn.Size(); rank++) if (n_on_rank[rank] != n_on_rank[0]) { mprinterr("Internal Error: Number of sets to sync on rank %i (%i) != number on master %i\n", rank, n_on_rank[rank], n_on_rank[0]); return 1; } // Send all data set sizes to master. std::vector<int> all_rank_sizes; if (commIn.Master()) { all_rank_sizes.resize( nSets * commIn.Size() ); commIn.GatherMaster( &size_on_rank[0], nSets, MPI_INT, &all_rank_sizes[0] ); } else { commIn.GatherMaster( &size_on_rank[0], nSets, MPI_INT, 0 ); } size_on_rank.clear(); // Call Sync only for sets that need it. std::vector<int> rank_frames( commIn.Size() ); int total = 0; //TODO size_t? int idx0 = 0; for (DataListType::iterator ds = SetsToSync.begin(); ds != SetsToSync.end(); ++ds, ++idx0) { if (commIn.Master()) { total = all_rank_sizes[idx0]; rank_frames[0] = all_rank_sizes[idx0]; int idx1 = idx0 + nSets; for (int rank = 1; rank < commIn.Size(); rank++, idx1 += nSets) { total += all_rank_sizes[idx1]; rank_frames[rank] = all_rank_sizes[idx1]; } //mprintf("DEBUG: Syncing '%s' (size=%zu, total=%i)\n", (*ds)->Meta().PrintName().c_str(), // (*ds)->Size(), total); } if ( (*ds)->Sync(total, rank_frames, commIn) ) { rprintf( "Warning: Could not sync dataset '%s'\n",(*ds)->legend()); //return; } (*ds)->SetNeedsSync( false ); } return 0; }