void Region::setDimensions(Dimensions& newDims) { // Can only set dimensions one time if (dims_ == newDims) return; if (dims_.isUnspecified()) { if (newDims.isDontcare()) { NTA_THROW << "Invalid attempt to set region dimensions to dontcare value"; } if (! newDims.isValid()) { NTA_THROW << "Attempt to set region dimensions to invalid value:" << newDims.toString(); } dims_ = newDims; dimensionInfo_ = "Specified explicitly in setDimensions()"; } else { NTA_THROW << "Attempt to set dimensions of region " << getName() << " to " << newDims.toString() << " but region already has dimensions " << dims_.toString(); } // can only create the enabled node set after we know the number of dimensions setupEnabledNodeSet(); }
void DCDataSet::create(const CollectionType& colType, hid_t group, const Dimensions size, uint32_t ndims, bool compression) throw (DCException) { log_msg(2, "DCDataSet::create (%s, size %s)", name.c_str(), size.toString().c_str()); if (opened) throw DCException(getExceptionString("create: dataset is already open")); // if the dataset already exists, remove/unlink it // note that this won't free the memory occupied by this // dataset, however, there currently is no function to delete // a dataset if (!checkExistence || (checkExistence && H5Lexists(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT))) H5Ldelete(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT); this->ndims = ndims; this->compression = compression; this->datatype = colType.getDataType(); getLogicalSize().set(size); setChunking(colType.getSize()); setCompression(); if (getPhysicalSize().getScalarSize() != 0) { hsize_t *max_dims = new hsize_t[ndims]; for (size_t i = 0; i < ndims; ++i) max_dims[i] = H5F_UNLIMITED; dataspace = H5Screate_simple(ndims, getPhysicalSize().getPointer(), max_dims); delete[] max_dims; max_dims = NULL; } else dataspace = H5Screate(H5S_NULL); if (dataspace < 0) throw DCException(getExceptionString("create: Failed to create dataspace")); // create the new dataset dataset = H5Dcreate(group, this->name.c_str(), this->datatype, dataspace, H5P_DEFAULT, dsetProperties, H5P_DEFAULT); if (dataset < 0) throw DCException(getExceptionString("create: Failed to create dataset")); isReference = false; opened = true; }
TEST_F(DimensionsTest, DontCareDimensions) { // dontcare dimensions [0] Dimensions d; d.push_back(0); ASSERT_TRUE(!d.isUnspecified()); ASSERT_TRUE(d.isDontcare()); ASSERT_TRUE(d.isValid()); EXPECT_STREQ("[dontcare]", d.toString().c_str()); ASSERT_ANY_THROW(d.getIndex(zero)); ASSERT_ANY_THROW(d.getCount()); ASSERT_EQ((unsigned int)0, d.getDimension(0)); ASSERT_EQ((unsigned int)1, d.getDimensionCount()); }
TEST_F(DimensionsTest, EmptyDimensions) { // empty dimensions (unspecified) Dimensions d; ASSERT_TRUE(d.isUnspecified()); ASSERT_TRUE(d.isValid()); ASSERT_TRUE(!d.isDontcare()); ASSERT_ANY_THROW(d.getCount()); ASSERT_ANY_THROW(d.getDimension(0)); EXPECT_STREQ("[unspecified]", d.toString().c_str()); ASSERT_ANY_THROW(d.getIndex(one_two)); ASSERT_ANY_THROW(d.getCount()); ASSERT_ANY_THROW(d.getDimension(0)); ASSERT_EQ((unsigned int)0, d.getDimensionCount()); }
TEST_F(DimensionsTest, InvalidDimensions) { // invalid dimensions Dimensions d; d.push_back(1); d.push_back(0); ASSERT_TRUE(!d.isUnspecified()); ASSERT_TRUE(!d.isDontcare()); ASSERT_TRUE(!d.isValid()); EXPECT_STREQ("[1 0] (invalid)", d.toString().c_str()); ASSERT_ANY_THROW(d.getIndex(one_two)); ASSERT_ANY_THROW(d.getCount()); ASSERT_EQ((unsigned int)1, d.getDimension(0)); ASSERT_EQ((unsigned int)0, d.getDimension(1)); ASSERT_ANY_THROW(d.getDimension(2)); ASSERT_EQ((unsigned int)2, d.getDimensionCount()); }
TEST_F(DimensionsTest, ValidDimensions) { // valid dimensions [2,3] // two rows, three columns Dimensions d; d.push_back(2); d.push_back(3); ASSERT_TRUE(!d.isUnspecified()); ASSERT_TRUE(!d.isDontcare()); ASSERT_TRUE(d.isValid()); EXPECT_STREQ("[2 3]", d.toString().c_str()); ASSERT_EQ((unsigned int)2, d.getDimension(0)); ASSERT_EQ((unsigned int)3, d.getDimension(1)); ASSERT_ANY_THROW(d.getDimension(2)); ASSERT_EQ((unsigned int)6, d.getCount()); ASSERT_EQ((unsigned int)5, d.getIndex(one_two)); ASSERT_EQ((unsigned int)2, d.getDimensionCount()); }
void DCDataSet::write(Dimensions srcBuffer, Dimensions srcStride, Dimensions srcOffset, Dimensions srcData, Dimensions dstOffset, const void* data) throw (DCException) { log_msg(2, "DCDataSet::write (%s)", name.c_str()); if (!opened) throw DCException(getExceptionString("write: Dataset has not been opened/created")); log_msg(3, " ndims = %llu\n" " logical_size = %s\n" " physical_size = %s\n" " src_buffer = %s\n" " src_stride = %s\n" " src_data = %s\n" " src_offset = %s\n" " dst_offset = %s\n", (long long unsigned) ndims, getLogicalSize().toString().c_str(), getPhysicalSize().toString().c_str(), srcBuffer.toString().c_str(), srcStride.toString().c_str(), srcData.toString().c_str(), srcOffset.toString().c_str(), dstOffset.toString().c_str()); // swap dimensions if necessary srcBuffer.swapDims(ndims); srcStride.swapDims(ndims); srcData.swapDims(ndims); srcOffset.swapDims(ndims); dstOffset.swapDims(ndims); // dataspace to read from hid_t dsp_src; if (getLogicalSize().getScalarSize() != 0) { dsp_src = H5Screate_simple(ndims, srcBuffer.getPointer(), NULL); if (dsp_src < 0) throw DCException(getExceptionString("write: Failed to create source dataspace")); if (H5Sselect_hyperslab(dsp_src, H5S_SELECT_SET, srcOffset.getPointer(), srcStride.getPointer(), srcData.getPointer(), NULL) < 0 || H5Sselect_valid(dsp_src) <= 0) throw DCException(getExceptionString("write: Invalid source hyperslap selection")); if (srcData.getScalarSize() == 0) H5Sselect_none(dsp_src); // dataspace to write to if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, dstOffset.getPointer(), NULL, srcData.getPointer(), NULL) < 0 || H5Sselect_valid(dataspace) <= 0) throw DCException(getExceptionString("write: Invalid target hyperslap selection")); if (!data || (srcData.getScalarSize() == 0)) { H5Sselect_none(dataspace); data = NULL; } // write data to the dataset if (H5Dwrite(dataset, this->datatype, dsp_src, dataspace, dsetWriteProperties, data) < 0) throw DCException(getExceptionString("write: Failed to write dataset")); H5Sclose(dsp_src); } }
void DCDataSet::read(Dimensions dstBuffer, Dimensions dstOffset, Dimensions srcSize, Dimensions srcOffset, Dimensions& sizeRead, uint32_t& srcNDims, void* dst) throw (DCException) { log_msg(2, "DCDataSet::read (%s)", name.c_str()); if (!opened) throw DCException(getExceptionString("read: Dataset has not been opened/created")); if (dstBuffer.getScalarSize() == 0) dstBuffer.set(srcSize); // dst buffer is allowed to be NULL // in this case, only the size of the dataset is returned // if the dataset is empty, return just its size as there is nothing to read if ((dst != NULL) && (getNDims() > 0)) { log_msg(3, " ndims = %llu\n" " logical_size = %s\n" " physical_size = %s\n" " dstBuffer = %s\n" " dstOffset = %s\n" " srcSize = %s\n" " srcOffset = %s\n", (long long unsigned) ndims, getLogicalSize().toString().c_str(), getPhysicalSize().toString().c_str(), dstBuffer.toString().c_str(), dstOffset.toString().c_str(), srcSize.toString().c_str(), srcOffset.toString().c_str()); dstBuffer.swapDims(ndims); dstOffset.swapDims(ndims); srcSize.swapDims(ndims); srcOffset.swapDims(ndims); hid_t dst_dataspace = H5Screate_simple(ndims, dstBuffer.getPointer(), NULL); if (dst_dataspace < 0) throw DCException(getExceptionString("read: Failed to create target dataspace")); if (H5Sselect_hyperslab(dst_dataspace, H5S_SELECT_SET, dstOffset.getPointer(), NULL, srcSize.getPointer(), NULL) < 0 || H5Sselect_valid(dst_dataspace) <= 0) throw DCException(getExceptionString("read: Target dataspace hyperslab selection is not valid!")); if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, srcOffset.getPointer(), NULL, srcSize.getPointer(), NULL) < 0 || H5Sselect_valid(dataspace) <= 0) throw DCException(getExceptionString("read: Source dataspace hyperslab selection is not valid!")); if (srcSize.getScalarSize() == 0) H5Sselect_none(dataspace); if (H5Dread(dataset, this->datatype, dst_dataspace, dataspace, dsetReadProperties, dst) < 0) throw DCException(getExceptionString("read: Failed to read dataset")); H5Sclose(dst_dataspace); srcSize.swapDims(ndims); } // swap dimensions if necessary sizeRead.set(srcSize); srcNDims = this->ndims; log_msg(3, " returns ndims = %llu", (long long unsigned) ndims); log_msg(3, " returns sizeRead = %s", sizeRead.toString().c_str()); }
int main(int argc, char **argv) { MPI_Init(&argc, &argv); if (argc < 2) { std::cout << "Usage: " << argv[0] << " <libsplash-file-base>" << std::endl; return -1; } int mpi_rank, mpi_size; int files_start = 0, files_end = 0; Dimensions file_mpi_size; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); // libSplash filename std::string filename; filename.assign(argv[1]); // create DomainCollector // read single file, not merged DomainCollector dc(100); DataCollector::FileCreationAttr fAttr; DataCollector::initFileCreationAttr(fAttr); fAttr.fileAccType = DataCollector::FAT_READ; // broadcast file MPI size from root to all processes uint64_t f_mpi_size[3]; if (mpi_rank == 0) { fAttr.mpiPosition.set(0, 0, 0); dc.open(filename.c_str(), fAttr); dc.getMPISize(file_mpi_size); std::cout << mpi_rank << ": total file MPI size = " << file_mpi_size.toString() << std::endl; for (int i = 0; i < 3; ++i) f_mpi_size[i] = file_mpi_size[i]; dc.close(); } MPI_Bcast(f_mpi_size, 3, MPI_UNSIGNED_LONG_LONG, 0, MPI_COMM_WORLD); file_mpi_size.set(f_mpi_size[0], f_mpi_size[1], f_mpi_size[2]); // get number of files for this MPI process filesToProcesses(mpi_size, mpi_rank, file_mpi_size.getScalarSize(), files_start, files_end); for (int f = files_start; f <= files_end; ++f) { // get file MPI pos for this file index indexToPos(f, file_mpi_size, fAttr.mpiPosition); std::cout << mpi_rank << ": opening position " << fAttr.mpiPosition.toString() << std::endl; dc.open(filename.c_str(), fAttr); // get number of entries int32_t *ids = NULL; size_t num_ids = 0; dc.getEntryIDs(NULL, &num_ids); if (num_ids == 0) { dc.close(); continue; } else { ids = new int32_t[num_ids]; dc.getEntryIDs(ids, &num_ids); } // get entries for 1. id (iteration) DataCollector::DCEntry *entries = NULL; size_t num_entries = 0; dc.getEntriesForID(ids[0], NULL, &num_entries); if (num_entries == 0) { delete[] ids; dc.close(); continue; } else { entries = new DataCollector::DCEntry[num_entries]; dc.getEntriesForID(ids[0], entries, &num_entries); } // read 1. entry DataCollector::DCEntry first_entry = entries[0]; std::cout << " " << mpi_rank << ": reading entry " << first_entry.name << std::endl; // read complete domain Domain domain = dc.getGlobalDomain(ids[0], first_entry.name.c_str()); DomainCollector::DomDataClass dataClass = DomainCollector::UndefinedType; DataContainer* container = dc.readDomain(ids[0], first_entry.name.c_str(), domain, &dataClass, false); // access all elements, no matter how many subdomains for (size_t i = 0; i < container->getNumElements(); ++i) { void *element = container->getElement(i); // do anything with this element // ... } // POLY data might be distributed over multiple subdomains for (size_t d = 0; d < container->getNumSubdomains(); ++d) { DomainData* subdomain = container->getIndex(d); Dimensions size = subdomain->getSize(); std::cout << " " << mpi_rank << ": subdomain " << d << " has size " << size.toString() << std::endl; // access the underlying buffer of a subdomain void *elements = subdomain->getData(); } // don't forget to delete the container allocated by DomainCollector delete container; delete[] entries; delete[] ids; dc.close(); } MPI_Finalize(); return 0; }
int main(int argc, char **argv) { if (argc < 2) { std::cout << "Usage: " << argv[0] << " <libsplash-file-base>" << std::endl; return -1; } // libSplash filename std::string filename; filename.assign(argv[1]); // create DomainCollector DomainCollector dc(100); DataCollector::FileCreationAttr fAttr; DataCollector::initFileCreationAttr(fAttr); fAttr.fileAccType = DataCollector::FAT_READ_MERGED; dc.open(filename.c_str(), fAttr); // get number of entries int32_t *ids = NULL; size_t num_ids = 0; dc.getEntryIDs(NULL, &num_ids); if (num_ids == 0) { dc.close(); return 1; } else { ids = new int32_t[num_ids]; dc.getEntryIDs(ids, &num_ids); } // get entries for 1. id (iteration) std::cout << "reading from iteration " << ids[0] << std::endl; DataCollector::DCEntry *entries = NULL; size_t num_entries = 0; dc.getEntriesForID(ids[0], NULL, &num_entries); if (num_entries == 0) { delete[] ids; dc.close(); return 1; } else { entries = new DataCollector::DCEntry[num_entries]; dc.getEntriesForID(ids[0], entries, &num_entries); } // read 1. entry from this iteration DataCollector::DCEntry first_entry = entries[0]; std::cout << "reading entry " << first_entry.name << std::endl; // read complete domain Domain domain = dc.getGlobalDomain(ids[0], first_entry.name.c_str()); DomainCollector::DomDataClass dataClass = DomainCollector::UndefinedType; DataContainer* container = dc.readDomain(ids[0], first_entry.name.c_str(), domain, &dataClass, false); // access all elements, no matter how many subdomains for (size_t i = 0; i < container->getNumElements(); ++i) { void *element = container->getElement(i); // do anything with this element // ... } // POLY data might be distributed over multiple subdomains for (size_t d = 0; d < container->getNumSubdomains(); ++d) { DomainData* subdomain = container->getIndex(d); Dimensions size = subdomain->getSize(); std::cout << "subdomain " << d << " has size " << size.toString() << std::endl; // access the underlying buffer of a subdomain void *elements = subdomain->getData(); } // don't forget to delete the container allocated by DomainCollector delete container; delete[] entries; delete[] ids; dc.close(); return 0; }
int main(int argc, const char * argv[]) { // Create network Network net = Network(); // Add VectorFileSensor region to network Region* region = net.addRegion("region", "VectorFileSensor", "{activeOutputCount: 1}"); // Set region dimensions Dimensions dims; dims.push_back(1); std::cout << "Setting region dimensions" << dims.toString() << std::endl; region->setDimensions(dims); // Load data std::string path = Path::makeAbsolute("../../src/examples/regions/Data.csv"); std::cout << "Loading data from " << path << std::endl; std::vector<std::string> loadFileArgs; loadFileArgs.push_back("loadFile"); loadFileArgs.push_back(path); loadFileArgs.push_back("2"); region->executeCommand(loadFileArgs); // Initialize network std::cout << "Initializing network" << std::endl; net.initialize(); ArrayRef outputArray = region->getOutputData("dataOut"); // Compute std::cout << "Compute" << std::endl; region->compute(); // Get output Real64 *buffer = (Real64*) outputArray.getBuffer(); for (size_t i = 0; i < outputArray.getCount(); i++) { std::cout << " " << i << " " << buffer[i] << "" << std::endl; } // Serialize Network net2; { std::stringstream ss; net.write(ss); net2.read(ss); } net2.initialize(); Region* region2 = net2.getRegions().getByName("region"); region2->executeCommand(loadFileArgs); ArrayRef outputArray2 = region2->getOutputData("dataOut"); Real64 *buffer2 = (Real64*)outputArray2.getBuffer(); net.run(1); net2.run(1); NTA_ASSERT(outputArray2.getCount() == outputArray.getCount()); for (size_t i = 0; i < outputArray.getCount(); i++) { std::cout << " " << i << " " << buffer[i] << " " << buffer2[i] << std::endl; } return 0; }
void DimensionsTest::RunTests() { Coordinate zero; // [0]; zero.push_back(0); Coordinate one_two; // [1,2] one_two.push_back(1); one_two.push_back(2); Coordinate three_four; // [3,4] three_four.push_back(3); three_four.push_back(4); { // empty dimensions (unspecified) Dimensions d; TEST(d.isUnspecified()); TEST(d.isValid()); TEST(!d.isDontcare()); SHOULDFAIL(d.getCount()); SHOULDFAIL(d.getDimension(0)); TESTEQUAL("[unspecified]", d.toString()); SHOULDFAIL(d.getIndex(one_two)); SHOULDFAIL(d.getCount()); SHOULDFAIL(d.getDimension(0)); TESTEQUAL((unsigned int)0, d.getDimensionCount()); } { // dontcare dimensions [0] Dimensions d; d.push_back(0); TEST(!d.isUnspecified()); TEST(d.isDontcare()); TEST(d.isValid()); TESTEQUAL("[dontcare]", d.toString()); SHOULDFAIL(d.getIndex(zero)); SHOULDFAIL(d.getCount()); TESTEQUAL((unsigned int)0, d.getDimension(0)); TESTEQUAL((unsigned int)1, d.getDimensionCount()); } { // invalid dimensions Dimensions d; d.push_back(1); d.push_back(0); TEST(!d.isUnspecified()); TEST(!d.isDontcare()); TEST(!d.isValid()); TESTEQUAL("[1 0] (invalid)", d.toString()); SHOULDFAIL(d.getIndex(one_two)); SHOULDFAIL(d.getCount()); TESTEQUAL((unsigned int)1, d.getDimension(0)); TESTEQUAL((unsigned int)0, d.getDimension(1)); SHOULDFAIL(d.getDimension(2)); TESTEQUAL((unsigned int)2, d.getDimensionCount()); } { // valid dimensions [2,3] // two rows, three columns Dimensions d; d.push_back(2); d.push_back(3); TEST(!d.isUnspecified()); TEST(!d.isDontcare()); TEST(d.isValid()); TESTEQUAL("[2 3]", d.toString()); TESTEQUAL((unsigned int)2, d.getDimension(0)); TESTEQUAL((unsigned int)3, d.getDimension(1)); SHOULDFAIL(d.getDimension(2)); TESTEQUAL((unsigned int)6, d.getCount()); TESTEQUAL((unsigned int)5, d.getIndex(one_two)); TESTEQUAL((unsigned int)2, d.getDimensionCount()); } { //check a two dimensional matrix for proper x-major ordering std::vector<size_t> x; x.push_back(4); x.push_back(5); Dimensions d(x); size_t testDim1 = 4; size_t testDim2 = 5; for(size_t i = 0; i < testDim1; i++) { for(size_t j = 0; j < testDim2; j++) { Coordinate testCoordinate; testCoordinate.push_back(i); testCoordinate.push_back(j); TESTEQUAL(i+j*testDim1, d.getIndex(testCoordinate)); TESTEQUAL(vecToString(testCoordinate), vecToString(d.getCoordinate(i+j*testDim1))); } } } { //check a three dimensional matrix for proper x-major ordering std::vector<size_t> x; x.push_back(3); x.push_back(4); x.push_back(5); Dimensions d(x); size_t testDim1 = 3; size_t testDim2 = 4; size_t testDim3 = 5; for(size_t i = 0; i < testDim1; i++) { for(size_t j = 0; j < testDim2; j++) { for(size_t k = 0; k < testDim3; k++) { Coordinate testCoordinate; testCoordinate.push_back(i); testCoordinate.push_back(j); testCoordinate.push_back(k); TESTEQUAL(i + j*testDim1 + k*testDim1*testDim2, d.getIndex(testCoordinate)); TESTEQUAL(vecToString(testCoordinate), vecToString(d.getCoordinate(i + j*testDim1 + k*testDim1*testDim2))); } } } } { // alternate constructor std::vector<size_t> x; x.push_back(2); x.push_back(5); Dimensions d(x); TEST(!d.isUnspecified()); TEST(!d.isDontcare()); TEST(d.isValid()); TESTEQUAL((unsigned int)2, d.getDimension(0)); TESTEQUAL((unsigned int)5, d.getDimension(1)); SHOULDFAIL(d.getDimension(2)); TESTEQUAL((unsigned int)2, d.getDimensionCount()); } } // RunTests()