Example #1
0
void
Region::setDimensions(Dimensions& newDims)
{
  // Can only set dimensions one time
  if (dims_ == newDims)
    return;
  
  if (dims_.isUnspecified())
  {
    if (newDims.isDontcare())
    {
      NTA_THROW << "Invalid attempt to set region dimensions to dontcare value";
    }

    if (! newDims.isValid())
    {
      NTA_THROW << "Attempt to set region dimensions to invalid value:"
                << newDims.toString();
    }

    dims_ = newDims;
    dimensionInfo_ = "Specified explicitly in setDimensions()";
  } else {
    NTA_THROW << "Attempt to set dimensions of region " << getName() 
              << " to " << newDims.toString()
              << " but region already has dimensions " << dims_.toString();
  }
  
  // can only create the enabled node set after we know the number of dimensions
  setupEnabledNodeSet();

}
Example #2
0
    void DCDataSet::create(const CollectionType& colType,
            hid_t group, const Dimensions size, uint32_t ndims, bool compression)
    throw (DCException)
    {
        log_msg(2, "DCDataSet::create (%s, size %s)", name.c_str(), size.toString().c_str());

        if (opened)
            throw DCException(getExceptionString("create: dataset is already open"));

        // if the dataset already exists, remove/unlink it
        // note that this won't free the memory occupied by this
        // dataset, however, there currently is no function to delete
        // a dataset
        if (!checkExistence || (checkExistence && H5Lexists(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT)))
            H5Ldelete(group, name.c_str(), H5P_LINK_ACCESS_DEFAULT);

        this->ndims = ndims;
        this->compression = compression;
        this->datatype = colType.getDataType();

        getLogicalSize().set(size);

        setChunking(colType.getSize());
        setCompression();

        if (getPhysicalSize().getScalarSize() != 0)
        {
            hsize_t *max_dims = new hsize_t[ndims];
            for (size_t i = 0; i < ndims; ++i)
                max_dims[i] = H5F_UNLIMITED;

            dataspace = H5Screate_simple(ndims, getPhysicalSize().getPointer(), max_dims);

            delete[] max_dims;
            max_dims = NULL;
        } else
            dataspace = H5Screate(H5S_NULL);



        if (dataspace < 0)
            throw DCException(getExceptionString("create: Failed to create dataspace"));

        // create the new dataset
        dataset = H5Dcreate(group, this->name.c_str(), this->datatype, dataspace,
                H5P_DEFAULT, dsetProperties, H5P_DEFAULT);

        if (dataset < 0)
            throw DCException(getExceptionString("create: Failed to create dataset"));

        isReference = false;
        opened = true;
    }
Example #3
0
TEST_F(DimensionsTest, DontCareDimensions) {
  // dontcare dimensions [0]
  Dimensions d;
  d.push_back(0);
  ASSERT_TRUE(!d.isUnspecified());
  ASSERT_TRUE(d.isDontcare());
  ASSERT_TRUE(d.isValid());
  EXPECT_STREQ("[dontcare]", d.toString().c_str());
  ASSERT_ANY_THROW(d.getIndex(zero));
  ASSERT_ANY_THROW(d.getCount());
  ASSERT_EQ((unsigned int)0, d.getDimension(0));
  ASSERT_EQ((unsigned int)1, d.getDimensionCount());
}
Example #4
0
TEST_F(DimensionsTest, EmptyDimensions) {
  // empty dimensions (unspecified)
  Dimensions d;
  ASSERT_TRUE(d.isUnspecified());
  ASSERT_TRUE(d.isValid());
  ASSERT_TRUE(!d.isDontcare());
  ASSERT_ANY_THROW(d.getCount());
  ASSERT_ANY_THROW(d.getDimension(0));
  EXPECT_STREQ("[unspecified]", d.toString().c_str());
  ASSERT_ANY_THROW(d.getIndex(one_two));
  ASSERT_ANY_THROW(d.getCount());
  ASSERT_ANY_THROW(d.getDimension(0));
  ASSERT_EQ((unsigned int)0, d.getDimensionCount());
}
Example #5
0
TEST_F(DimensionsTest, InvalidDimensions) {
  // invalid dimensions
  Dimensions d;
  d.push_back(1);
  d.push_back(0);
  ASSERT_TRUE(!d.isUnspecified());
  ASSERT_TRUE(!d.isDontcare());
  ASSERT_TRUE(!d.isValid());
  EXPECT_STREQ("[1 0] (invalid)", d.toString().c_str());
  ASSERT_ANY_THROW(d.getIndex(one_two));
  ASSERT_ANY_THROW(d.getCount());
  ASSERT_EQ((unsigned int)1, d.getDimension(0));
  ASSERT_EQ((unsigned int)0, d.getDimension(1));
  ASSERT_ANY_THROW(d.getDimension(2));
  ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
Example #6
0
TEST_F(DimensionsTest, ValidDimensions) {
  // valid dimensions [2,3]
  // two rows, three columns
  Dimensions d;
  d.push_back(2);
  d.push_back(3);
  ASSERT_TRUE(!d.isUnspecified());
  ASSERT_TRUE(!d.isDontcare());
  ASSERT_TRUE(d.isValid());
  EXPECT_STREQ("[2 3]", d.toString().c_str());
  ASSERT_EQ((unsigned int)2, d.getDimension(0));
  ASSERT_EQ((unsigned int)3, d.getDimension(1));
  ASSERT_ANY_THROW(d.getDimension(2));
  ASSERT_EQ((unsigned int)6, d.getCount());
  ASSERT_EQ((unsigned int)5, d.getIndex(one_two));
  ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
Example #7
0
    void DCDataSet::write(Dimensions srcBuffer, Dimensions srcStride,
            Dimensions srcOffset, Dimensions srcData,
            Dimensions dstOffset, const void* data)
    throw (DCException)
    {
        log_msg(2, "DCDataSet::write (%s)", name.c_str());

        if (!opened)
            throw DCException(getExceptionString("write: Dataset has not been opened/created"));

        log_msg(3,
                " ndims = %llu\n"
                " logical_size = %s\n"
                " physical_size = %s\n"
                " src_buffer = %s\n"
                " src_stride = %s\n"
                " src_data = %s\n"
                " src_offset = %s\n"
                " dst_offset = %s\n",
                (long long unsigned) ndims,
                getLogicalSize().toString().c_str(),
                getPhysicalSize().toString().c_str(),
                srcBuffer.toString().c_str(),
                srcStride.toString().c_str(),
                srcData.toString().c_str(),
                srcOffset.toString().c_str(),
                dstOffset.toString().c_str());

        // swap dimensions if necessary
        srcBuffer.swapDims(ndims);
        srcStride.swapDims(ndims);
        srcData.swapDims(ndims);
        srcOffset.swapDims(ndims);
        dstOffset.swapDims(ndims);

        // dataspace to read from
        hid_t dsp_src;

        if (getLogicalSize().getScalarSize() != 0)
        {
            dsp_src = H5Screate_simple(ndims, srcBuffer.getPointer(), NULL);
            if (dsp_src < 0)
                throw DCException(getExceptionString("write: Failed to create source dataspace"));

            if (H5Sselect_hyperslab(dsp_src, H5S_SELECT_SET, srcOffset.getPointer(),
                    srcStride.getPointer(), srcData.getPointer(), NULL) < 0 ||
                    H5Sselect_valid(dsp_src) <= 0)
                throw DCException(getExceptionString("write: Invalid source hyperslap selection"));

            if (srcData.getScalarSize() == 0)
                H5Sselect_none(dsp_src);

            // dataspace to write to
            if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, dstOffset.getPointer(),
                    NULL, srcData.getPointer(), NULL) < 0 ||
                    H5Sselect_valid(dataspace) <= 0)
                throw DCException(getExceptionString("write: Invalid target hyperslap selection"));

            if (!data || (srcData.getScalarSize() == 0))
            {
                H5Sselect_none(dataspace);
                data = NULL;
            }

            // write data to the dataset

            if (H5Dwrite(dataset, this->datatype, dsp_src, dataspace, dsetWriteProperties, data) < 0)
                throw DCException(getExceptionString("write: Failed to write dataset"));

            H5Sclose(dsp_src);
        }
    }
Example #8
0
    void DCDataSet::read(Dimensions dstBuffer,
            Dimensions dstOffset,
            Dimensions srcSize,
            Dimensions srcOffset,
            Dimensions& sizeRead,
            uint32_t& srcNDims,
            void* dst)
    throw (DCException)
    {
        log_msg(2, "DCDataSet::read (%s)", name.c_str());

        if (!opened)
            throw DCException(getExceptionString("read: Dataset has not been opened/created"));

        if (dstBuffer.getScalarSize() == 0)
            dstBuffer.set(srcSize);

        // dst buffer is allowed to be NULL
        // in this case, only the size of the dataset is returned
        // if the dataset is empty, return just its size as there is nothing to read
        if ((dst != NULL) && (getNDims() > 0))
        {
            log_msg(3,
                    " ndims = %llu\n"
                    " logical_size = %s\n"
                    " physical_size = %s\n"
                    " dstBuffer = %s\n"
                    " dstOffset = %s\n"
                    " srcSize = %s\n"
                    " srcOffset = %s\n",
                    (long long unsigned) ndims,
                    getLogicalSize().toString().c_str(),
                    getPhysicalSize().toString().c_str(),
                    dstBuffer.toString().c_str(),
                    dstOffset.toString().c_str(),
                    srcSize.toString().c_str(),
                    srcOffset.toString().c_str());

            dstBuffer.swapDims(ndims);
            dstOffset.swapDims(ndims);
            srcSize.swapDims(ndims);
            srcOffset.swapDims(ndims);

            hid_t dst_dataspace = H5Screate_simple(ndims, dstBuffer.getPointer(), NULL);
            if (dst_dataspace < 0)
                throw DCException(getExceptionString("read: Failed to create target dataspace"));

            if (H5Sselect_hyperslab(dst_dataspace, H5S_SELECT_SET, dstOffset.getPointer(), NULL,
                    srcSize.getPointer(), NULL) < 0 ||
                    H5Sselect_valid(dst_dataspace) <= 0)
                throw DCException(getExceptionString("read: Target dataspace hyperslab selection is not valid!"));

            if (H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, srcOffset.getPointer(), NULL,
                    srcSize.getPointer(), NULL) < 0 ||
                    H5Sselect_valid(dataspace) <= 0)
                throw DCException(getExceptionString("read: Source dataspace hyperslab selection is not valid!"));

            if (srcSize.getScalarSize() == 0)
                H5Sselect_none(dataspace);
            
            if (H5Dread(dataset, this->datatype, dst_dataspace, dataspace, dsetReadProperties, dst) < 0)
                throw DCException(getExceptionString("read: Failed to read dataset"));

            H5Sclose(dst_dataspace);

            srcSize.swapDims(ndims);
        }

        // swap dimensions if necessary
        sizeRead.set(srcSize);
        srcNDims = this->ndims;

        log_msg(3, " returns ndims = %llu", (long long unsigned) ndims);
        log_msg(3, " returns sizeRead = %s", sizeRead.toString().c_str());
    }
int main(int argc, char **argv)
{
    MPI_Init(&argc, &argv);

    if (argc < 2)
    {
        std::cout << "Usage: " << argv[0] << " <libsplash-file-base>" << std::endl;
        return -1;
    }

    int mpi_rank, mpi_size;
    int files_start = 0, files_end = 0;
    Dimensions file_mpi_size;
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);

    // libSplash filename
    std::string filename;
    filename.assign(argv[1]);

    // create DomainCollector
    // read single file, not merged
    DomainCollector dc(100);
    DataCollector::FileCreationAttr fAttr;
    DataCollector::initFileCreationAttr(fAttr);
    fAttr.fileAccType = DataCollector::FAT_READ;

    // broadcast file MPI size from root to all processes
    uint64_t f_mpi_size[3];
    if (mpi_rank == 0)
    {
        fAttr.mpiPosition.set(0, 0, 0);
        dc.open(filename.c_str(), fAttr);

        dc.getMPISize(file_mpi_size);
        std::cout << mpi_rank << ": total file MPI size = " <<
                file_mpi_size.toString() << std::endl;

        for (int i = 0; i < 3; ++i)
            f_mpi_size[i] = file_mpi_size[i];

        dc.close();
    }
    MPI_Bcast(f_mpi_size, 3, MPI_UNSIGNED_LONG_LONG, 0, MPI_COMM_WORLD);
    file_mpi_size.set(f_mpi_size[0], f_mpi_size[1], f_mpi_size[2]);

    // get number of files for this MPI process
    filesToProcesses(mpi_size, mpi_rank, file_mpi_size.getScalarSize(),
            files_start, files_end);

    for (int f = files_start; f <= files_end; ++f)
    {
        // get file MPI pos for this file index
        indexToPos(f, file_mpi_size, fAttr.mpiPosition);
        std::cout << mpi_rank << ": opening position " <<
                fAttr.mpiPosition.toString() << std::endl;

        dc.open(filename.c_str(), fAttr);

        // get number of entries
        int32_t *ids = NULL;
        size_t num_ids = 0;
        dc.getEntryIDs(NULL, &num_ids);

        if (num_ids == 0)
        {
            dc.close();
            continue;
        } else
        {
            ids = new int32_t[num_ids];
            dc.getEntryIDs(ids, &num_ids);
        }

        // get entries for 1. id (iteration)
        DataCollector::DCEntry *entries = NULL;
        size_t num_entries = 0;
        dc.getEntriesForID(ids[0], NULL, &num_entries);

        if (num_entries == 0)
        {
            delete[] ids;
            dc.close();
            continue;
        } else
        {
            entries = new DataCollector::DCEntry[num_entries];
            dc.getEntriesForID(ids[0], entries, &num_entries);
        }

        // read 1. entry
        DataCollector::DCEntry first_entry = entries[0];
        std::cout << "  " << mpi_rank << ": reading entry " << first_entry.name << std::endl;

        // read complete domain
        Domain domain = dc.getGlobalDomain(ids[0], first_entry.name.c_str());
        DomainCollector::DomDataClass dataClass = DomainCollector::UndefinedType;
        DataContainer* container = dc.readDomain(ids[0], first_entry.name.c_str(),
                domain, &dataClass, false);

        // access all elements, no matter how many subdomains
        for (size_t i = 0; i < container->getNumElements(); ++i)
        {
            void *element = container->getElement(i);
            // do anything with this element
            // ...
        }

        // POLY data might be distributed over multiple subdomains
        for (size_t d = 0; d < container->getNumSubdomains(); ++d)
        {
            DomainData* subdomain = container->getIndex(d);
            Dimensions size = subdomain->getSize();

            std::cout << "  " << mpi_rank << ": subdomain " << d << " has size " <<
                    size.toString() << std::endl;

            // access the underlying buffer of a subdomain
            void *elements = subdomain->getData();
        }

        // don't forget to delete the container allocated by DomainCollector
        delete container;

        delete[] entries;
        delete[] ids;

        dc.close();
    }

    MPI_Finalize();

    return 0;
}
int main(int argc, char **argv)
{
    if (argc < 2)
    {
        std::cout << "Usage: " << argv[0] << " <libsplash-file-base>" << std::endl;
        return -1;
    }

    // libSplash filename
    std::string filename;
    filename.assign(argv[1]);

    // create DomainCollector
    DomainCollector dc(100);
    DataCollector::FileCreationAttr fAttr;
    DataCollector::initFileCreationAttr(fAttr);

    fAttr.fileAccType = DataCollector::FAT_READ_MERGED;

    dc.open(filename.c_str(), fAttr);

    // get number of entries
    int32_t *ids = NULL;
    size_t num_ids = 0;
    dc.getEntryIDs(NULL, &num_ids);

    if (num_ids == 0)
    {
        dc.close();
        return 1;
    } else
    {
        ids = new int32_t[num_ids];
        dc.getEntryIDs(ids, &num_ids);
    }

    // get entries for 1. id (iteration)
    std::cout << "reading from iteration " << ids[0] << std::endl;
    DataCollector::DCEntry *entries = NULL;
    size_t num_entries = 0;
    dc.getEntriesForID(ids[0], NULL, &num_entries);

    if (num_entries == 0)
    {
        delete[] ids;
        dc.close();
        return 1;
    } else
    {
        entries = new DataCollector::DCEntry[num_entries];
        dc.getEntriesForID(ids[0], entries, &num_entries);
    }

    // read 1. entry from this iteration
    DataCollector::DCEntry first_entry = entries[0];
    std::cout << "reading entry " << first_entry.name << std::endl;

    // read complete domain
    Domain domain = dc.getGlobalDomain(ids[0], first_entry.name.c_str());
    DomainCollector::DomDataClass dataClass = DomainCollector::UndefinedType;
    DataContainer* container = dc.readDomain(ids[0], first_entry.name.c_str(),
            domain, &dataClass, false);

    // access all elements, no matter how many subdomains
    for (size_t i = 0; i < container->getNumElements(); ++i)
    {
        void *element = container->getElement(i);
        // do anything with this element
        // ...
    }

    // POLY data might be distributed over multiple subdomains
    for (size_t d = 0; d < container->getNumSubdomains(); ++d)
    {
        DomainData* subdomain = container->getIndex(d);
        Dimensions size = subdomain->getSize();

        std::cout << "subdomain " << d << " has size " << size.toString() << std::endl;

        // access the underlying buffer of a subdomain
        void *elements = subdomain->getData();
    }

    // don't forget to delete the container allocated by DomainCollector
    delete container;

    delete[] entries;
    delete[] ids;

    dc.close();

    return 0;
}
int main(int argc, const char * argv[])
{
    // Create network
    Network net = Network();

    // Add VectorFileSensor region to network
    Region* region = net.addRegion("region", "VectorFileSensor", "{activeOutputCount: 1}");

    // Set region dimensions
    Dimensions dims;
    dims.push_back(1);

    std::cout << "Setting region dimensions" << dims.toString() << std::endl;

    region->setDimensions(dims);

    // Load data
    std::string path = Path::makeAbsolute("../../src/examples/regions/Data.csv");

    std::cout << "Loading data from " << path << std::endl;

    std::vector<std::string> loadFileArgs;
    loadFileArgs.push_back("loadFile");
    loadFileArgs.push_back(path);
    loadFileArgs.push_back("2");

    region->executeCommand(loadFileArgs);

    // Initialize network
    std::cout << "Initializing network" << std::endl;

    net.initialize();

    ArrayRef outputArray = region->getOutputData("dataOut");

    // Compute
    std::cout << "Compute" << std::endl;

    region->compute();

    // Get output
    Real64 *buffer = (Real64*) outputArray.getBuffer();

    for (size_t i = 0; i < outputArray.getCount(); i++)
    {
        std::cout << "  " << i << "    " << buffer[i] << "" << std::endl;
    }

    // Serialize
    Network net2;
    {
      std::stringstream ss;
      net.write(ss);
      net2.read(ss);
    }
    net2.initialize();

    Region* region2 = net2.getRegions().getByName("region");
    region2->executeCommand(loadFileArgs);
    ArrayRef outputArray2 = region2->getOutputData("dataOut");
    Real64 *buffer2 = (Real64*)outputArray2.getBuffer();

    net.run(1);
    net2.run(1);

    NTA_ASSERT(outputArray2.getCount() == outputArray.getCount());
    for (size_t i = 0; i < outputArray.getCount(); i++)
    {
        std::cout << "  " << i << "    " << buffer[i] << "   " << buffer2[i]
                  << std::endl;
    }

    return 0;
}
Example #12
0
  void DimensionsTest::RunTests()
  {
    Coordinate zero; // [0];
    zero.push_back(0);

    Coordinate one_two; // [1,2]
    one_two.push_back(1);
    one_two.push_back(2);

    Coordinate three_four; // [3,4]
    three_four.push_back(3);
    three_four.push_back(4);
    

    {
      // empty dimensions (unspecified)
      Dimensions d;
      TEST(d.isUnspecified());
      TEST(d.isValid());
      TEST(!d.isDontcare());
      SHOULDFAIL(d.getCount());
      SHOULDFAIL(d.getDimension(0));
      TESTEQUAL("[unspecified]", d.toString());
      SHOULDFAIL(d.getIndex(one_two));
      SHOULDFAIL(d.getCount());
      SHOULDFAIL(d.getDimension(0));
      TESTEQUAL((unsigned int)0, d.getDimensionCount());
    }

    {
      // dontcare dimensions [0]
      Dimensions d;
      d.push_back(0);
      TEST(!d.isUnspecified());
      TEST(d.isDontcare());
      TEST(d.isValid());
      TESTEQUAL("[dontcare]", d.toString());
      SHOULDFAIL(d.getIndex(zero));
      SHOULDFAIL(d.getCount());
      TESTEQUAL((unsigned int)0, d.getDimension(0));
      TESTEQUAL((unsigned int)1, d.getDimensionCount());
    }


    {
      // invalid dimensions
      Dimensions d;
      d.push_back(1);
      d.push_back(0);
      TEST(!d.isUnspecified());
      TEST(!d.isDontcare());
      TEST(!d.isValid());
      TESTEQUAL("[1 0] (invalid)", d.toString());
      SHOULDFAIL(d.getIndex(one_two));
      SHOULDFAIL(d.getCount());
      TESTEQUAL((unsigned int)1, d.getDimension(0));
      TESTEQUAL((unsigned int)0, d.getDimension(1));
      SHOULDFAIL(d.getDimension(2));
      TESTEQUAL((unsigned int)2, d.getDimensionCount());
    }

    {
      // valid dimensions [2,3]
      // two rows, three columns
      Dimensions d;
      d.push_back(2);
      d.push_back(3);
      TEST(!d.isUnspecified());
      TEST(!d.isDontcare());
      TEST(d.isValid());
      TESTEQUAL("[2 3]", d.toString());
      TESTEQUAL((unsigned int)2, d.getDimension(0));
      TESTEQUAL((unsigned int)3, d.getDimension(1));
      SHOULDFAIL(d.getDimension(2));
      TESTEQUAL((unsigned int)6, d.getCount());
      TESTEQUAL((unsigned int)5, d.getIndex(one_two));
      TESTEQUAL((unsigned int)2, d.getDimensionCount());
    }

    {
      //check a two dimensional matrix for proper x-major ordering
      std::vector<size_t> x;
      x.push_back(4);
      x.push_back(5);
      Dimensions d(x);
      size_t testDim1 = 4;
      size_t testDim2 = 5;
      for(size_t i = 0; i < testDim1; i++)
      {
        for(size_t j = 0; j < testDim2; j++)
        {
          Coordinate testCoordinate;
          testCoordinate.push_back(i);
          testCoordinate.push_back(j);

          TESTEQUAL(i+j*testDim1, d.getIndex(testCoordinate));
          TESTEQUAL(vecToString(testCoordinate),
                    vecToString(d.getCoordinate(i+j*testDim1)));
        }
      }
    }

    {
      //check a three dimensional matrix for proper x-major ordering
      std::vector<size_t> x;
      x.push_back(3);
      x.push_back(4);
      x.push_back(5);
      Dimensions d(x);
      size_t testDim1 = 3;
      size_t testDim2 = 4;
      size_t testDim3 = 5;
      for(size_t i = 0; i < testDim1; i++)
      {
        for(size_t j = 0; j < testDim2; j++)
        {
          for(size_t k = 0; k < testDim3; k++)
          {
            Coordinate testCoordinate;
            testCoordinate.push_back(i);
            testCoordinate.push_back(j);
            testCoordinate.push_back(k);

            TESTEQUAL(i +
                      j*testDim1 +
                      k*testDim1*testDim2, d.getIndex(testCoordinate));

            TESTEQUAL(vecToString(testCoordinate),
                      vecToString(d.getCoordinate(i +
                                                  j*testDim1 +
                                                  k*testDim1*testDim2)));
          }
        }
      }
    }

    { 
      // alternate constructor
      std::vector<size_t> x;
      x.push_back(2);
      x.push_back(5);
      Dimensions d(x);
      TEST(!d.isUnspecified());
      TEST(!d.isDontcare());
      TEST(d.isValid());
      
      TESTEQUAL((unsigned int)2, d.getDimension(0));
      TESTEQUAL((unsigned int)5, d.getDimension(1));
      SHOULDFAIL(d.getDimension(2));
      TESTEQUAL((unsigned int)2, d.getDimensionCount());
    }

  } // RunTests()