void
Region::setDimensions(Dimensions& newDims)
{
// Can only set dimensions one time
if (dims_ == newDims)
return;
if (dims_.isUnspecified())
{
if (newDims.isDontcare())
{
NTA_THROW << "Invalid attempt to set region dimensions to dontcare value";
}
if (! newDims.isValid())
{
NTA_THROW << "Attempt to set region dimensions to invalid value:"
<< newDims.toString();
}
dims_ = newDims;
dimensionInfo_ = "Specified explicitly in setDimensions()";
} else {
NTA_THROW << "Attempt to set dimensions of region " << getName()
<< " to " << newDims.toString()
<< " but region already has dimensions " << dims_.toString();
}
// can only create the enabled node set after we know the number of dimensions
setupEnabledNodeSet();
}
void DCDataSet::create(const CollectionType& colType,
hid_t group, const Dimensions size, uint32_t ndims, bool compression)
throw (DCException)
{
log_msg(2, "DCDataSet::create (%s, size %s)", name.c_str(), size.toString().c_str());
if (opened)
throw DCException(getExceptionString("create: dataset is already open"));
// if the dataset already exists, remove/unlink it
// note that this won't free the memory occupied by this
// dataset, however, there currently is no function to delete
// a dataset
if (!checkExistence || (checkExistence && H5Lexists(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT)))
H5Ldelete(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT);
this->ndims = ndims;
this->compression = compression;
this->datatype = colType.getDataType();
getLogicalSize().set(size);
setChunking(colType.getSize());
setCompression();
if (getPhysicalSize().getScalarSize() != 0)
{
hsize_t *max_dims = new hsize_t[ndims];
for (size_t i = 0; i < ndims; ++i)
max_dims[i] = H5F_UNLIMITED;
dataspace = H5Screate_simple(ndims, getPhysicalSize().getPointer(), max_dims);
delete[] max_dims;
max_dims = NULL;
} else
dataspace = H5Screate(H5S_NULL);
if (dataspace < 0)
throw DCException(getExceptionString("create: Failed to create dataspace"));
// create the new dataset
dataset = H5Dcreate(group, this->name.c_str(), this->datatype, dataspace,
H5P_DEFAULT, dsetProperties, H5P_DEFAULT);
if (dataset < 0)
throw DCException(getExceptionString("create: Failed to create dataset"));
isReference = false;
opened = true;
}
TEST_F(DimensionsTest, DontCareDimensions) {
// dontcare dimensions [0]
Dimensions d;
d.push_back(0);
ASSERT_TRUE(!d.isUnspecified());
ASSERT_TRUE(d.isDontcare());
ASSERT_TRUE(d.isValid());
EXPECT_STREQ("[dontcare]", d.toString().c_str());
ASSERT_ANY_THROW(d.getIndex(zero));
ASSERT_ANY_THROW(d.getCount());
ASSERT_EQ((unsigned int)0, d.getDimension(0));
ASSERT_EQ((unsigned int)1, d.getDimensionCount());
}
TEST_F(DimensionsTest, EmptyDimensions) {
// empty dimensions (unspecified)
Dimensions d;
ASSERT_TRUE(d.isUnspecified());
ASSERT_TRUE(d.isValid());
ASSERT_TRUE(!d.isDontcare());
ASSERT_ANY_THROW(d.getCount());
ASSERT_ANY_THROW(d.getDimension(0));
EXPECT_STREQ("[unspecified]", d.toString().c_str());
ASSERT_ANY_THROW(d.getIndex(one_two));
ASSERT_ANY_THROW(d.getCount());
ASSERT_ANY_THROW(d.getDimension(0));
ASSERT_EQ((unsigned int)0, d.getDimensionCount());
}
TEST_F(DimensionsTest, InvalidDimensions) {
// invalid dimensions
Dimensions d;
d.push_back(1);
d.push_back(0);
ASSERT_TRUE(!d.isUnspecified());
ASSERT_TRUE(!d.isDontcare());
ASSERT_TRUE(!d.isValid());
EXPECT_STREQ("[1 0] (invalid)", d.toString().c_str());
ASSERT_ANY_THROW(d.getIndex(one_two));
ASSERT_ANY_THROW(d.getCount());
ASSERT_EQ((unsigned int)1, d.getDimension(0));
ASSERT_EQ((unsigned int)0, d.getDimension(1));
ASSERT_ANY_THROW(d.getDimension(2));
ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
TEST_F(DimensionsTest, ValidDimensions) {
// valid dimensions [2,3]
// two rows, three columns
Dimensions d;
d.push_back(2);
d.push_back(3);
ASSERT_TRUE(!d.isUnspecified());
ASSERT_TRUE(!d.isDontcare());
ASSERT_TRUE(d.isValid());
EXPECT_STREQ("[2 3]", d.toString().c_str());
ASSERT_EQ((unsigned int)2, d.getDimension(0));
ASSERT_EQ((unsigned int)3, d.getDimension(1));
ASSERT_ANY_THROW(d.getDimension(2));
ASSERT_EQ((unsigned int)6, d.getCount());
ASSERT_EQ((unsigned int)5, d.getIndex(one_two));
ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
void DCDataSet::write(Dimensions srcBuffer, Dimensions srcStride,
Dimensions srcOffset, Dimensions srcData,
Dimensions dstOffset, const void* data)
throw (DCException)
{
log_msg(2, "DCDataSet::write (%s)", name.c_str());
if (!opened)
throw DCException(getExceptionString("write: Dataset has not been opened/created"));
log_msg(3,
" ndims = %llu\n"
" logical_size = %s\n"
" physical_size = %s\n"
" src_buffer = %s\n"
" src_stride = %s\n"
" src_data = %s\n"
" src_offset = %s\n"
" dst_offset = %s\n",
(long long unsigned) ndims,
getLogicalSize().toString().c_str(),
getPhysicalSize().toString().c_str(),
srcBuffer.toString().c_str(),
srcStride.toString().c_str(),
srcData.toString().c_str(),
srcOffset.toString().c_str(),
dstOffset.toString().c_str());
// swap dimensions if necessary
srcBuffer.swapDims(ndims);
srcStride.swapDims(ndims);
srcData.swapDims(ndims);
srcOffset.swapDims(ndims);
dstOffset.swapDims(ndims);
// dataspace to read from
hid_t dsp_src;
if (getLogicalSize().getScalarSize() != 0)
{
dsp_src = H5Screate_simple(ndims, srcBuffer.getPointer(), NULL);
if (dsp_src < 0)
throw DCException(getExceptionString("write: Failed to create source dataspace"));
if (H5Sselect_hyperslab(dsp_src, H5S_SELECT_SET, srcOffset.getPointer(),
srcStride.getPointer(), srcData.getPointer(), NULL) < 0 ||
H5Sselect_valid(dsp_src) <= 0)
throw DCException(getExceptionString("write: Invalid source hyperslap selection"));
if (srcData.getScalarSize() == 0)
H5Sselect_none(dsp_src);
// dataspace to write to
if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, dstOffset.getPointer(),
NULL, srcData.getPointer(), NULL) < 0 ||
H5Sselect_valid(dataspace) <= 0)
throw DCException(getExceptionString("write: Invalid target hyperslap selection"));
if (!data || (srcData.getScalarSize() == 0))
{
H5Sselect_none(dataspace);
data = NULL;
}
// write data to the dataset
if (H5Dwrite(dataset, this->datatype, dsp_src, dataspace, dsetWriteProperties, data) < 0)
throw DCException(getExceptionString("write: Failed to write dataset"));
H5Sclose(dsp_src);
}
}
void DCDataSet::read(Dimensions dstBuffer,
Dimensions dstOffset,
Dimensions srcSize,
Dimensions srcOffset,
Dimensions& sizeRead,
uint32_t& srcNDims,
void* dst)
throw (DCException)
{
log_msg(2, "DCDataSet::read (%s)", name.c_str());
if (!opened)
throw DCException(getExceptionString("read: Dataset has not been opened/created"));
if (dstBuffer.getScalarSize() == 0)
dstBuffer.set(srcSize);
// dst buffer is allowed to be NULL
// in this case, only the size of the dataset is returned
// if the dataset is empty, return just its size as there is nothing to read
if ((dst != NULL) && (getNDims() > 0))
{
log_msg(3,
" ndims = %llu\n"
" logical_size = %s\n"
" physical_size = %s\n"
" dstBuffer = %s\n"
" dstOffset = %s\n"
" srcSize = %s\n"
" srcOffset = %s\n",
(long long unsigned) ndims,
getLogicalSize().toString().c_str(),
getPhysicalSize().toString().c_str(),
dstBuffer.toString().c_str(),
dstOffset.toString().c_str(),
srcSize.toString().c_str(),
srcOffset.toString().c_str());
dstBuffer.swapDims(ndims);
dstOffset.swapDims(ndims);
srcSize.swapDims(ndims);
srcOffset.swapDims(ndims);
hid_t dst_dataspace = H5Screate_simple(ndims, dstBuffer.getPointer(), NULL);
if (dst_dataspace < 0)
throw DCException(getExceptionString("read: Failed to create target dataspace"));
if (H5Sselect_hyperslab(dst_dataspace, H5S_SELECT_SET, dstOffset.getPointer(), NULL,
srcSize.getPointer(), NULL) < 0 ||
H5Sselect_valid(dst_dataspace) <= 0)
throw DCException(getExceptionString("read: Target dataspace hyperslab selection is not valid!"));
if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, srcOffset.getPointer(), NULL,
srcSize.getPointer(), NULL) < 0 ||
H5Sselect_valid(dataspace) <= 0)
throw DCException(getExceptionString("read: Source dataspace hyperslab selection is not valid!"));
if (srcSize.getScalarSize() == 0)
H5Sselect_none(dataspace);
if (H5Dread(dataset, this->datatype, dst_dataspace, dataspace, dsetReadProperties, dst) < 0)
throw DCException(getExceptionString("read: Failed to read dataset"));
H5Sclose(dst_dataspace);
srcSize.swapDims(ndims);
}
// swap dimensions if necessary
sizeRead.set(srcSize);
srcNDims = this->ndims;
log_msg(3, " returns ndims = %llu", (long long unsigned) ndims);
log_msg(3, " returns sizeRead = %s", sizeRead.toString().c_str());
}
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);
if (argc < 2)
{
std::cout << "Usage: " << argv[0] << " <libsplash-file-base>" << std::endl;
return -1;
}
int mpi_rank, mpi_size;
int files_start = 0, files_end = 0;
Dimensions file_mpi_size;
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
// libSplash filename
std::string filename;
filename.assign(argv[1]);
// create DomainCollector
// read single file, not merged
DomainCollector dc(100);
DataCollector::FileCreationAttr fAttr;
DataCollector::initFileCreationAttr(fAttr);
fAttr.fileAccType = DataCollector::FAT_READ;
// broadcast file MPI size from root to all processes
uint64_t f_mpi_size[3];
if (mpi_rank == 0)
{
fAttr.mpiPosition.set(0, 0, 0);
dc.open(filename.c_str(), fAttr);
dc.getMPISize(file_mpi_size);
std::cout << mpi_rank << ": total file MPI size = " <<
file_mpi_size.toString() << std::endl;
for (int i = 0; i < 3; ++i)
f_mpi_size[i] = file_mpi_size[i];
dc.close();
}
MPI_Bcast(f_mpi_size, 3, MPI_UNSIGNED_LONG_LONG, 0, MPI_COMM_WORLD);
file_mpi_size.set(f_mpi_size[0], f_mpi_size[1], f_mpi_size[2]);
// get number of files for this MPI process
filesToProcesses(mpi_size, mpi_rank, file_mpi_size.getScalarSize(),
files_start, files_end);
for (int f = files_start; f <= files_end; ++f)
{
// get file MPI pos for this file index
indexToPos(f, file_mpi_size, fAttr.mpiPosition);
std::cout << mpi_rank << ": opening position " <<
fAttr.mpiPosition.toString() << std::endl;
dc.open(filename.c_str(), fAttr);
// get number of entries
int32_t *ids = NULL;
size_t num_ids = 0;
dc.getEntryIDs(NULL, &num_ids);
if (num_ids == 0)
{
dc.close();
continue;
} else
{
ids = new int32_t[num_ids];
dc.getEntryIDs(ids, &num_ids);
}
// get entries for 1. id (iteration)
DataCollector::DCEntry *entries = NULL;
size_t num_entries = 0;
dc.getEntriesForID(ids[0], NULL, &num_entries);
if (num_entries == 0)
{
delete[] ids;
dc.close();
continue;
} else
{
entries = new DataCollector::DCEntry[num_entries];
dc.getEntriesForID(ids[0], entries, &num_entries);
}
// read 1. entry
DataCollector::DCEntry first_entry = entries[0];
std::cout << " " << mpi_rank << ": reading entry " << first_entry.name << std::endl;
// read complete domain
Domain domain = dc.getGlobalDomain(ids[0], first_entry.name.c_str());
DomainCollector::DomDataClass dataClass = DomainCollector::UndefinedType;
DataContainer* container = dc.readDomain(ids[0], first_entry.name.c_str(),
domain, &dataClass, false);
// access all elements, no matter how many subdomains
for (size_t i = 0; i < container->getNumElements(); ++i)
{
void *element = container->getElement(i);
// do anything with this element
// ...
}
// POLY data might be distributed over multiple subdomains
for (size_t d = 0; d < container->getNumSubdomains(); ++d)
{
DomainData* subdomain = container->getIndex(d);
Dimensions size = subdomain->getSize();
std::cout << " " << mpi_rank << ": subdomain " << d << " has size " <<
size.toString() << std::endl;
// access the underlying buffer of a subdomain
void *elements = subdomain->getData();
}
// don't forget to delete the container allocated by DomainCollector
delete container;
delete[] entries;
delete[] ids;
dc.close();
}
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
if (argc < 2)
{
std::cout << "Usage: " << argv[0] << " <libsplash-file-base>" << std::endl;
return -1;
}
// libSplash filename
std::string filename;
filename.assign(argv[1]);
// create DomainCollector
DomainCollector dc(100);
DataCollector::FileCreationAttr fAttr;
DataCollector::initFileCreationAttr(fAttr);
fAttr.fileAccType = DataCollector::FAT_READ_MERGED;
dc.open(filename.c_str(), fAttr);
// get number of entries
int32_t *ids = NULL;
size_t num_ids = 0;
dc.getEntryIDs(NULL, &num_ids);
if (num_ids == 0)
{
dc.close();
return 1;
} else
{
ids = new int32_t[num_ids];
dc.getEntryIDs(ids, &num_ids);
}
// get entries for 1. id (iteration)
std::cout << "reading from iteration " << ids[0] << std::endl;
DataCollector::DCEntry *entries = NULL;
size_t num_entries = 0;
dc.getEntriesForID(ids[0], NULL, &num_entries);
if (num_entries == 0)
{
delete[] ids;
dc.close();
return 1;
} else
{
entries = new DataCollector::DCEntry[num_entries];
dc.getEntriesForID(ids[0], entries, &num_entries);
}
// read 1. entry from this iteration
DataCollector::DCEntry first_entry = entries[0];
std::cout << "reading entry " << first_entry.name << std::endl;
// read complete domain
Domain domain = dc.getGlobalDomain(ids[0], first_entry.name.c_str());
DomainCollector::DomDataClass dataClass = DomainCollector::UndefinedType;
DataContainer* container = dc.readDomain(ids[0], first_entry.name.c_str(),
domain, &dataClass, false);
// access all elements, no matter how many subdomains
for (size_t i = 0; i < container->getNumElements(); ++i)
{
void *element = container->getElement(i);
// do anything with this element
// ...
}
// POLY data might be distributed over multiple subdomains
for (size_t d = 0; d < container->getNumSubdomains(); ++d)
{
DomainData* subdomain = container->getIndex(d);
Dimensions size = subdomain->getSize();
std::cout << "subdomain " << d << " has size " << size.toString() << std::endl;
// access the underlying buffer of a subdomain
void *elements = subdomain->getData();
}
// don't forget to delete the container allocated by DomainCollector
delete container;
delete[] entries;
delete[] ids;
dc.close();
return 0;
}
int main(int argc, const char * argv[])
{
// Create network
Network net = Network();
// Add VectorFileSensor region to network
Region* region = net.addRegion("region", "VectorFileSensor", "{activeOutputCount: 1}");
// Set region dimensions
Dimensions dims;
dims.push_back(1);
std::cout << "Setting region dimensions" << dims.toString() << std::endl;
region->setDimensions(dims);
// Load data
std::string path = Path::makeAbsolute("../../src/examples/regions/Data.csv");
std::cout << "Loading data from " << path << std::endl;
std::vector<std::string> loadFileArgs;
loadFileArgs.push_back("loadFile");
loadFileArgs.push_back(path);
loadFileArgs.push_back("2");
region->executeCommand(loadFileArgs);
// Initialize network
std::cout << "Initializing network" << std::endl;
net.initialize();
ArrayRef outputArray = region->getOutputData("dataOut");
// Compute
std::cout << "Compute" << std::endl;
region->compute();
// Get output
Real64 *buffer = (Real64*) outputArray.getBuffer();
for (size_t i = 0; i < outputArray.getCount(); i++)
{
std::cout << " " << i << " " << buffer[i] << "" << std::endl;
}
// Serialize
Network net2;
{
std::stringstream ss;
net.write(ss);
net2.read(ss);
}
net2.initialize();
Region* region2 = net2.getRegions().getByName("region");
region2->executeCommand(loadFileArgs);
ArrayRef outputArray2 = region2->getOutputData("dataOut");
Real64 *buffer2 = (Real64*)outputArray2.getBuffer();
net.run(1);
net2.run(1);
NTA_ASSERT(outputArray2.getCount() == outputArray.getCount());
for (size_t i = 0; i < outputArray.getCount(); i++)
{
std::cout << " " << i << " " << buffer[i] << " " << buffer2[i]
<< std::endl;
}
return 0;
}
void DimensionsTest::RunTests()
{
Coordinate zero; // [0];
zero.push_back(0);
Coordinate one_two; // [1,2]
one_two.push_back(1);
one_two.push_back(2);
Coordinate three_four; // [3,4]
three_four.push_back(3);
three_four.push_back(4);
{
// empty dimensions (unspecified)
Dimensions d;
TEST(d.isUnspecified());
TEST(d.isValid());
TEST(!d.isDontcare());
SHOULDFAIL(d.getCount());
SHOULDFAIL(d.getDimension(0));
TESTEQUAL("[unspecified]", d.toString());
SHOULDFAIL(d.getIndex(one_two));
SHOULDFAIL(d.getCount());
SHOULDFAIL(d.getDimension(0));
TESTEQUAL((unsigned int)0, d.getDimensionCount());
}
{
// dontcare dimensions [0]
Dimensions d;
d.push_back(0);
TEST(!d.isUnspecified());
TEST(d.isDontcare());
TEST(d.isValid());
TESTEQUAL("[dontcare]", d.toString());
SHOULDFAIL(d.getIndex(zero));
SHOULDFAIL(d.getCount());
TESTEQUAL((unsigned int)0, d.getDimension(0));
TESTEQUAL((unsigned int)1, d.getDimensionCount());
}
{
// invalid dimensions
Dimensions d;
d.push_back(1);
d.push_back(0);
TEST(!d.isUnspecified());
TEST(!d.isDontcare());
TEST(!d.isValid());
TESTEQUAL("[1 0] (invalid)", d.toString());
SHOULDFAIL(d.getIndex(one_two));
SHOULDFAIL(d.getCount());
TESTEQUAL((unsigned int)1, d.getDimension(0));
TESTEQUAL((unsigned int)0, d.getDimension(1));
SHOULDFAIL(d.getDimension(2));
TESTEQUAL((unsigned int)2, d.getDimensionCount());
}
{
// valid dimensions [2,3]
// two rows, three columns
Dimensions d;
d.push_back(2);
d.push_back(3);
TEST(!d.isUnspecified());
TEST(!d.isDontcare());
TEST(d.isValid());
TESTEQUAL("[2 3]", d.toString());
TESTEQUAL((unsigned int)2, d.getDimension(0));
TESTEQUAL((unsigned int)3, d.getDimension(1));
SHOULDFAIL(d.getDimension(2));
TESTEQUAL((unsigned int)6, d.getCount());
TESTEQUAL((unsigned int)5, d.getIndex(one_two));
TESTEQUAL((unsigned int)2, d.getDimensionCount());
}
{
//check a two dimensional matrix for proper x-major ordering
std::vector<size_t> x;
x.push_back(4);
x.push_back(5);
Dimensions d(x);
size_t testDim1 = 4;
size_t testDim2 = 5;
for(size_t i = 0; i < testDim1; i++)
{
for(size_t j = 0; j < testDim2; j++)
{
Coordinate testCoordinate;
testCoordinate.push_back(i);
testCoordinate.push_back(j);
TESTEQUAL(i+j*testDim1, d.getIndex(testCoordinate));
TESTEQUAL(vecToString(testCoordinate),
vecToString(d.getCoordinate(i+j*testDim1)));
}
}
}
{
//check a three dimensional matrix for proper x-major ordering
std::vector<size_t> x;
x.push_back(3);
x.push_back(4);
x.push_back(5);
Dimensions d(x);
size_t testDim1 = 3;
size_t testDim2 = 4;
size_t testDim3 = 5;
for(size_t i = 0; i < testDim1; i++)
{
for(size_t j = 0; j < testDim2; j++)
{
for(size_t k = 0; k < testDim3; k++)
{
Coordinate testCoordinate;
testCoordinate.push_back(i);
testCoordinate.push_back(j);
testCoordinate.push_back(k);
TESTEQUAL(i +
j*testDim1 +
k*testDim1*testDim2, d.getIndex(testCoordinate));
TESTEQUAL(vecToString(testCoordinate),
vecToString(d.getCoordinate(i +
j*testDim1 +
k*testDim1*testDim2)));
}
}
}
}
{
// alternate constructor
std::vector<size_t> x;
x.push_back(2);
x.push_back(5);
Dimensions d(x);
TEST(!d.isUnspecified());
TEST(!d.isDontcare());
TEST(d.isValid());
TESTEQUAL((unsigned int)2, d.getDimension(0));
TESTEQUAL((unsigned int)5, d.getDimension(1));
SHOULDFAIL(d.getDimension(2));
TESTEQUAL((unsigned int)2, d.getDimensionCount());
}
} // RunTests()