ArrayDesc inferSchema(std::vector<ArrayDesc> schemas, boost::shared_ptr<Query> query)
{
Attributes atts;
atts.push_back(AttributeDesc(0, "attribute_name", TID_STRING, 0, 0));
atts.push_back(AttributeDesc(1, "min", TID_STRING, 0, 0));
atts.push_back(AttributeDesc(2, "max", TID_STRING, 0, 0));
atts.push_back(AttributeDesc(3, "distinct_count", TID_UINT64, 0, 0));
atts.push_back(AttributeDesc(4, "non_null_count", TID_UINT64, 0, 0));
const AttributeDesc *emptyIndicator = schemas[0].getEmptyBitmapAttribute();
set<string> a_s;
for (size_t i = 0; i < _parameters.size(); i++)
{
string attName = ((boost::shared_ptr<OperatorParamReference>&)_parameters[i])->getObjectName();
if (emptyIndicator && emptyIndicator->getName() == attName)
continue;
a_s.insert(attName);
}
size_t attsCount = (a_s.size() == 0 ? (emptyIndicator ? schemas[0].getAttributes().size() - 1 : schemas[0].getAttributes().size()) : a_s.size()) - 1;
Dimensions dims;
dims.push_back(DimensionDesc("attribute_number", 0, attsCount, ANALYZE_CHUNK_SIZE, 0));
return ArrayDesc(schemas[0].getName() + "_analyze", atts, dims);
}
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query)
{
Attributes outputAttrs;
outputAttrs.push_back(AttributeDesc(0, "dummy", TID_DOUBLE, AttributeDesc::IS_NULLABLE, 0));
Dimensions outputDims;
outputDims.push_back(DimensionDesc("i",0,0,1,0));
return ArrayDesc("test_cache", outputAttrs, outputDims);
}
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query)
{
assert(schemas.size() == 2);
ArrayDesc const& patternDesc = schemas[0];
ArrayDesc const& catalogDesc = schemas[1];
Attributes const& catalogAttributes = catalogDesc.getAttributes(true);
Dimensions const& catalogDimensions = catalogDesc.getDimensions();
Attributes const& patternAttributes = patternDesc.getAttributes(true);
Dimensions resultDimensions = patternDesc.getDimensions();
size_t totalAttributes = catalogAttributes.size() + patternAttributes.size() + 1 + catalogDimensions.size();
Attributes matchAttributes(totalAttributes);
if (catalogDimensions.size() != resultDimensions.size())
{
stringstream left, right;
printDimNames(left, resultDimensions);
printDimNames(right, catalogDimensions);
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_DIMENSION_COUNT_MISMATCH)
<< "match" << left.str() << right.str();
}
for (size_t i = 0, n = catalogDimensions.size(); i < n; i++) {
if (!(catalogDimensions[i].getStartMin() == resultDimensions[i].getStartMin()
&& catalogDimensions[i].getChunkInterval() == resultDimensions[i].getChunkInterval()
&& catalogDimensions[i].getChunkOverlap() == resultDimensions[i].getChunkOverlap()))
{
// XXX To do: implement requiresRepart() method, remove interval/overlap checks
// above, use SCIDB_LE_START_INDEX_MISMATCH here.
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_ARRAYS_NOT_CONFORMANT);
}
}
size_t j = 0;
for (size_t i = 0, n = patternAttributes.size(); i < n; i++, j++) {
AttributeDesc const& attr = patternAttributes[i];
matchAttributes[j] = AttributeDesc(j, attr.getName(), attr.getType(), attr.getFlags(),
attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(),
attr.getDefaultValueExpr());
}
for (size_t i = 0, n = catalogAttributes.size(); i < n; i++, j++) {
AttributeDesc const& attr = catalogAttributes[i];
matchAttributes[j] = AttributeDesc(j, "match_" + attr.getName(), attr.getType(), attr.getFlags(),
attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(),
attr.getDefaultValueExpr());
}
for (size_t i = 0, n = catalogDimensions.size(); i < n; i++, j++) {
matchAttributes[j] = AttributeDesc(j, "match_" + catalogDimensions[i].getBaseName(), TID_INT64, 0, 0);
}
matchAttributes[j] = AttributeDesc(j, DEFAULT_EMPTY_TAG_ATTRIBUTE_NAME, TID_INDICATOR, AttributeDesc::IS_EMPTY_INDICATOR, 0);
int64_t maxCollisions = evaluate(((boost::shared_ptr<OperatorParamLogicalExpression>&)_parameters[1])->getExpression(),
query, TID_INT64).getInt64();
if (maxCollisions <= 0 || (int32_t)maxCollisions != maxCollisions) {
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_OPERATOR_ARGUMENT2) << "positive";
}
resultDimensions.push_back(DimensionDesc("collision", 0, 0, maxCollisions-1, maxCollisions-1, (uint32_t)maxCollisions, 0));
return ArrayDesc("match", matchAttributes, resultDimensions);
}
TEST_F(DimensionsTest, DontCareDimensions) {
// dontcare dimensions [0]
Dimensions d;
d.push_back(0);
ASSERT_TRUE(!d.isUnspecified());
ASSERT_TRUE(d.isDontcare());
ASSERT_TRUE(d.isValid());
EXPECT_STREQ("[dontcare]", d.toString().c_str());
ASSERT_ANY_THROW(d.getIndex(zero));
ASSERT_ANY_THROW(d.getCount());
ASSERT_EQ((unsigned int)0, d.getDimension(0));
ASSERT_EQ((unsigned int)1, d.getDimensionCount());
}
TEST_F(DimensionsTest, InvalidDimensions) {
// invalid dimensions
Dimensions d;
d.push_back(1);
d.push_back(0);
ASSERT_TRUE(!d.isUnspecified());
ASSERT_TRUE(!d.isDontcare());
ASSERT_TRUE(!d.isValid());
EXPECT_STREQ("[1 0] (invalid)", d.toString().c_str());
ASSERT_ANY_THROW(d.getIndex(one_two));
ASSERT_ANY_THROW(d.getCount());
ASSERT_EQ((unsigned int)1, d.getDimension(0));
ASSERT_EQ((unsigned int)0, d.getDimension(1));
ASSERT_ANY_THROW(d.getDimension(2));
ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
TEST_F(DimensionsTest, ValidDimensions) {
// valid dimensions [2,3]
// two rows, three columns
Dimensions d;
d.push_back(2);
d.push_back(3);
ASSERT_TRUE(!d.isUnspecified());
ASSERT_TRUE(!d.isDontcare());
ASSERT_TRUE(d.isValid());
EXPECT_STREQ("[2 3]", d.toString().c_str());
ASSERT_EQ((unsigned int)2, d.getDimension(0));
ASSERT_EQ((unsigned int)3, d.getDimension(1));
ASSERT_ANY_THROW(d.getDimension(2));
ASSERT_EQ((unsigned int)6, d.getCount());
ASSERT_EQ((unsigned int)5, d.getIndex(one_two));
ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query)
{
assert(schemas.size() == 1);
ArrayDesc const& srcDesc = schemas[0];
ArrayDesc dstDesc = ((boost::shared_ptr<OperatorParamSchema>&)_parameters[0])->getSchema();
//Compile a desc of all possible attributes (aggregate calls first) and source dimensions
ArrayDesc aggregationDesc (srcDesc.getName(), Attributes(), srcDesc.getDimensions());
vector<string> aggregatedNames;
//add aggregate calls first
for (size_t i = 1; i < _parameters.size(); i++)
{
addAggregatedAttribute( (shared_ptr <OperatorParamAggregateCall>&) _parameters[i], srcDesc, aggregationDesc);
string aggName = aggregationDesc.getAttributes()[aggregationDesc.getAttributes().size()-1].getName();
bool aggFound = false;
BOOST_FOREACH(const AttributeDesc &dstAttr, dstDesc.getAttributes()) {
if (dstAttr.getName() == aggName) {
aggFound = true;
break;
}
}
if (!aggFound) {
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_ATTRIBUTE_DOESNT_EXIST) << aggName << dstDesc.getName();
}
aggregatedNames.push_back(aggName);
}
//add other attributes
BOOST_FOREACH(const AttributeDesc &srcAttr, srcDesc.getAttributes())
{
//if there's an attribute with same name as an aggregate call - skip the attribute
bool found = false;
BOOST_FOREACH(const AttributeDesc &aggAttr, aggregationDesc.getAttributes())
{
if( aggAttr.getName() == srcAttr.getName())
{
found = true;
}
}
if (!found)
{
aggregationDesc.addAttribute(AttributeDesc( aggregationDesc.getAttributes().size(),
srcAttr.getName(),
srcAttr.getType(),
srcAttr.getFlags(),
srcAttr.getDefaultCompressionMethod(),
srcAttr.getAliases(),
&srcAttr.getDefaultValue(),
srcAttr.getDefaultValueExpr(),
srcAttr.getVarSize()));
}
}
//Ensure attributes names uniqueness.
if (!dstDesc.getEmptyBitmapAttribute())
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OP_REDIMENSION_ERROR1);
BOOST_FOREACH(const AttributeDesc &dstAttr, dstDesc.getAttributes())
{
BOOST_FOREACH(const AttributeDesc &srcAttr, aggregationDesc.getAttributes())
{
if (srcAttr.getName() == dstAttr.getName())
{
if (srcAttr.getType() != dstAttr.getType())
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_ATTRIBUTE_TYPE)
<< srcAttr.getName() << srcAttr.getType() << dstAttr.getType();
}
if (!dstAttr.isNullable() && srcAttr.isNullable())
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_ATTRIBUTE_FLAGS)
<< srcAttr.getName();
}
goto NextAttr;
}
}
BOOST_FOREACH(const DimensionDesc &srcDim, aggregationDesc.getDimensions())
{
if (srcDim.hasNameAndAlias(dstAttr.getName()))
{
if (dstAttr.getType() != TID_INT64)
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_DESTINATION_ATTRIBUTE_TYPE)
<< dstAttr.getName() << TID_INT64;
}
if (dstAttr.getFlags() != 0)
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_DESTINATION_ATTRIBUTE_FLAGS)
<< dstAttr.getName();
}
goto NextAttr;
}
}
if (dstAttr.isEmptyIndicator() == false)
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_UNEXPECTED_DESTINATION_ATTRIBUTE)
<< dstAttr.getName();
}
NextAttr:;
}
Dimensions outputDims;
size_t nNewDims = 0;
BOOST_FOREACH(const DimensionDesc &dstDim, dstDesc.getDimensions())
{
if (dstDim.getChunkOverlap() > dstDim.getChunkInterval())
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OVERLAP_CANT_BE_LARGER_CHUNK);
}
BOOST_FOREACH(const AttributeDesc &srcAttr, aggregationDesc.getAttributes())
{
if (dstDim.hasNameAndAlias(srcAttr.getName()))
{
for (size_t i = 0; i< aggregatedNames.size(); i++)
{
if (srcAttr.getName() == aggregatedNames[i])
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OP_REDIMENSION_ERROR2);
}
if ( !IS_INTEGRAL(srcAttr.getType()) || srcAttr.getType() == TID_UINT64 )
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_SOURCE_ATTRIBUTE_TYPE)
<< srcAttr.getName() << TID_INT64;
}
outputDims.push_back(dstDim);
goto NextDim;
}
}
BOOST_FOREACH(const DimensionDesc &srcDim, aggregationDesc.getDimensions())
{
if (srcDim.hasNameAndAlias(dstDim.getBaseName()))
{
DimensionDesc outputDim = dstDim;
outputDims.push_back(outputDim);
goto NextDim;
}
}
//one synthetic dimension allowed
if (nNewDims++ != 0 || !aggregatedNames.empty() )
{
throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_UNEXPECTED_DESTINATION_DIMENSION) << dstDim.getBaseName();
}
outputDims.push_back(dstDim);
NextDim:;
}
return ArrayDesc(srcDesc.getName(), dstDesc.getAttributes(), outputDims, dstDesc.getFlags());
}
void Link::initialize(size_t destinationOffset)
{
// Make sure all information is specified and
// consistent. Unless there is a NuPIC implementation
// error, all these checks are guaranteed to pass
// because of the way the network is constructed
// and initialized.
// Make sure we have been attached to a real network
NTA_CHECK(src_ != nullptr);
NTA_CHECK(dest_ != nullptr);
// Confirm that our dimensions are consistent with the
// dimensions of the regions we're connecting.
const Dimensions& srcD = getSrcDimensions();
const Dimensions& destD = getDestDimensions();
NTA_CHECK(! srcD.isUnspecified());
NTA_CHECK(! destD.isUnspecified());
Dimensions oneD;
oneD.push_back(1);
if(src_->isRegionLevel())
{
Dimensions d;
for(size_t i = 0; i < src_->getRegion().getDimensions().size(); i++)
{
d.push_back(1);
}
NTA_CHECK(srcD.isDontcare() || srcD == d);
}
else if(src_->getRegion().getDimensions() == oneD)
{
Dimensions d;
for(size_t i = 0; i < srcD.size(); i++)
{
d.push_back(1);
}
NTA_CHECK(srcD.isDontcare() || srcD == d);
}
else
{
NTA_CHECK(srcD.isDontcare() || srcD == src_->getRegion().getDimensions());
}
if(dest_->isRegionLevel())
{
Dimensions d;
for(size_t i = 0; i < dest_->getRegion().getDimensions().size(); i++)
{
d.push_back(1);
}
NTA_CHECK(destD.isDontcare() || destD.isOnes());
}
else if(dest_->getRegion().getDimensions() == oneD)
{
Dimensions d;
for(size_t i = 0; i < destD.size(); i++)
{
d.push_back(1);
}
NTA_CHECK(destD.isDontcare() || destD == d);
}
else
{
NTA_CHECK(destD.isDontcare() || destD == dest_->getRegion().getDimensions());
}
destOffset_ = destinationOffset;
impl_->initialize();
initialized_ = true;
}
void RegionTest::testWithNodeType(const std::string& nodeType)
{
Region *rP = NULL;
SHOULDFAIL ( rP = new Region("r1", "nosuchnode", "") );
Region r("r1", nodeType, "");
TEST(r.getName() == "r1");
TEST(r.getType() == nodeType);
Dimensions d = r.getDimensions();
TEST(d.isUnspecified());
d.clear();
d.push_back(3);
d.push_back(2);
r.setDimensions(d);
Dimensions d2 = r.getDimensions();
TEST(d2.size() == 2);
TEST(d2[0] == 3);
TEST(d2[1] == 2);
TEST(d2.getCount() == 6);
// Parameter testing
{
{
Int32 val = -(1 << 24);
TESTEQUAL((Int32)32, r.getParameterInt32("int32Param"));
r.setParameterInt32("int32Param", val);
TESTEQUAL(val, r.getParameterInt32("int32Param"));
}
{
UInt32 val = 1 << 24;
TESTEQUAL((UInt32)33, r.getParameterUInt32("uint32Param"));
r.setParameterUInt32("uint32Param", val);
TESTEQUAL(val, r.getParameterUInt32("uint32Param"));
}
{
Int64 val = -((Int64)1 << 44);
TESTEQUAL((Int64)64, r.getParameterInt64("int64Param"));
r.setParameterInt64("int64Param", val);
TESTEQUAL(val, r.getParameterInt64("int64Param"));
}
{
UInt64 val = (UInt64)1 << 45;
TESTEQUAL((UInt64)65, r.getParameterUInt64("uint64Param"));
r.setParameterUInt64("uint64Param", val);
TESTEQUAL(r.getParameterUInt64("uint64Param"), val);
}
{
Real32 val = 23456.7;
TESTEQUAL((Real32)32.1, r.getParameterReal32("real32Param"));
r.setParameterReal32("real32Param", val);
TESTEQUAL(r.getParameterReal32("real32Param"), val);
}
{
Real64 val = 23456.789;
TESTEQUAL((Real64)64.1, r.getParameterReal64("real64Param"));
r.setParameterReal64("real64Param", val);
TESTEQUAL(r.getParameterReal64("real64Param"), val);
}
{
Array a(NTA_BasicType_Int64);
r.getParameterArray("int64ArrayParam", a);
// check default values
TESTEQUAL((size_t)4, a.getCount());
Int64 *buf = (Int64*) a.getBuffer();
TEST(buf != NULL);
for (UInt64 i = 0; i < 4; i++)
TESTEQUAL((Int64)(i*64), buf[i]);
// set our own value
buf[0] = 100;
r.setParameterArray("int64ArrayParam", a);
// make sure we retrieve the value just set
buf[0] = 0;
r.getParameterArray("int64ArrayParam", a);
TEST(buf == a.getBuffer());
TESTEQUAL((Int64)100, buf[0]);
}
{
std::string s = r.getParameterString("stringParam");
TESTEQUAL("nodespec value", s);
s = "new value";
r.setParameterString("stringParam", s);
s = r.getParameterString("stringParam");
TESTEQUAL("new value", s);
}
}
}
int main(int argc, const char * argv[])
{
// Create network
Network net = Network();
// Add VectorFileSensor region to network
Region* region = net.addRegion("region", "VectorFileSensor", "{activeOutputCount: 1}");
// Set region dimensions
Dimensions dims;
dims.push_back(1);
std::cout << "Setting region dimensions" << dims.toString() << std::endl;
region->setDimensions(dims);
// Load data
std::string path = Path::makeAbsolute("../../src/examples/regions/Data.csv");
std::cout << "Loading data from " << path << std::endl;
std::vector<std::string> loadFileArgs;
loadFileArgs.push_back("loadFile");
loadFileArgs.push_back(path);
loadFileArgs.push_back("2");
region->executeCommand(loadFileArgs);
// Initialize network
std::cout << "Initializing network" << std::endl;
net.initialize();
ArrayRef outputArray = region->getOutputData("dataOut");
// Compute
std::cout << "Compute" << std::endl;
region->compute();
// Get output
Real64 *buffer = (Real64*) outputArray.getBuffer();
for (size_t i = 0; i < outputArray.getCount(); i++)
{
std::cout << " " << i << " " << buffer[i] << "" << std::endl;
}
// Serialize
Network net2;
{
std::stringstream ss;
net.write(ss);
net2.read(ss);
}
net2.initialize();
Region* region2 = net2.getRegions().getByName("region");
region2->executeCommand(loadFileArgs);
ArrayRef outputArray2 = region2->getOutputData("dataOut");
Real64 *buffer2 = (Real64*)outputArray2.getBuffer();
net.run(1);
net2.run(1);
NTA_ASSERT(outputArray2.getCount() == outputArray.getCount());
for (size_t i = 0; i < outputArray.getCount(); i++)
{
std::cout << " " << i << " " << buffer[i] << " " << buffer2[i]
<< std::endl;
}
return 0;
}
int realmain(bool leakTest)
{
// verbose == true turns on extra output that is useful for
// debugging the test (e.g. when the TestNode compute()
// algorithm changes)
std::cout << "Creating network..." << std::endl;
Network n;
std::cout << "Region count is " << n.getRegions().getCount() << "" << std::endl;
std::cout << "Adding a PyNode region..." << std::endl;
Network::registerPyRegion("nupic.regions.TestNode", "TestNode");
Region* level2 = n.addRegion("level2", "py.TestNode", "{int32Param: 444}");
std::cout << "Region count is " << n.getRegions().getCount() << "" << std::endl;
std::cout << "Node type: " << level2->getType() << "" << std::endl;
std::cout << "Nodespec is:\n" << level2->getSpec()->toString() << "" << std::endl;
Real64 rval;
std::string int64Param("int64Param");
std::string real64Param("real64Param");
// get the value of intArrayParam after the setParameter call.
// --- Test getParameterReal64 of a PyNode
rval = level2->getParameterReal64("real64Param");
NTA_CHECK(rval == 64.1);
std::cout << "level2 getParameterReal64() returned: " << rval << std::endl;
// --- Test setParameterReal64 of a PyNode
level2->setParameterReal64("real64Param", 77.7);
rval = level2->getParameterReal64("real64Param");
NTA_CHECK(rval == 77.7);
// should fail because network has not been initialized
SHOULDFAIL(n.run(1));
// should fail because network can't be initialized
SHOULDFAIL (n.initialize() );
std::cout << "Setting dimensions of level1..." << std::endl;
Dimensions d;
d.push_back(4);
d.push_back(4);
std::cout << "Setting dimensions of level2..." << std::endl;
level2->setDimensions(d);
std::cout << "Initializing again..." << std::endl;
n.initialize();
testExceptionBug();
testPynodeInputOutputAccess(level2);
testPynodeArrayParameters(level2);
testPynodeLinking();
if (!leakTest)
{
//testNuPIC1x();
//testPynode1xLinking();
}
std::cout << "Done -- all tests passed" << std::endl;
return 0;
}
void Network::loadFromBundle(const std::string& name)
{
if (! StringUtils::endsWith(name, ".nta"))
NTA_THROW << "loadFromBundle: bundle extension must be \".nta\"";
std::string fullPath = Path::normalize(Path::makeAbsolute(name));
if (! Path::exists(fullPath))
NTA_THROW << "Path " << fullPath << " does not exist";
std::string networkStructureFilename = Path::join(fullPath, "network.yaml");
std::ifstream f(networkStructureFilename.c_str());
YAML::Parser parser(f);
YAML::Node doc;
bool success = parser.GetNextDocument(doc);
if (!success)
NTA_THROW << "Unable to find YAML document in network structure file "
<< networkStructureFilename;
if (doc.Type() != YAML::NodeType::Map)
NTA_THROW << "Invalid network structure file -- does not contain a map";
// Should contain Version, Regions, Links
if (doc.size() != 3)
NTA_THROW << "Invalid network structure file -- contains "
<< doc.size() << " elements";
// Extra version
const YAML::Node *node = doc.FindValue("Version");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- no version";
int version;
*node >> version;
if (version != 2)
NTA_THROW << "Invalid network structure file -- only version 2 supported";
// Regions
const YAML::Node *regions = doc.FindValue("Regions");
if (regions == NULL)
NTA_THROW << "Invalid network structure file -- no regions";
if (regions->Type() != YAML::NodeType::Sequence)
NTA_THROW << "Invalid network structure file -- regions element is not a list";
for (YAML::Iterator region = regions->begin(); region != regions->end(); region++)
{
// Each region is a map -- extract the 5 values in the map
if ((*region).Type() != YAML::NodeType::Map)
NTA_THROW << "Invalid network structure file -- bad region (not a map)";
if ((*region).size() != 5)
NTA_THROW << "Invalid network structure file -- bad region (wrong size)";
// 1. name
node = (*region).FindValue("name");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- region has no name";
std::string name;
*node >> name;
// 2. nodeType
node = (*region).FindValue("nodeType");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- region "
<< name << " has no node type";
std::string nodeType;
*node >> nodeType;
// 3. dimensions
node = (*region).FindValue("dimensions");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- region "
<< name << " has no dimensions";
if ((*node).Type() != YAML::NodeType::Sequence)
NTA_THROW << "Invalid network structure file -- region "
<< name << " dimensions specified incorrectly";
Dimensions dimensions;
for (YAML::Iterator valiter = (*node).begin(); valiter != (*node).end(); valiter++)
{
size_t val;
(*valiter) >> val;
dimensions.push_back(val);
}
// 4. phases
node = (*region).FindValue("phases");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- region"
<< name << "has no phases";
if ((*node).Type() != YAML::NodeType::Sequence)
NTA_THROW << "Invalid network structure file -- region "
<< name << " phases specified incorrectly";
std::set<UInt32> phases;
for (YAML::Iterator valiter = (*node).begin(); valiter != (*node).end(); valiter++)
{
UInt32 val;
(*valiter) >> val;
phases.insert(val);
}
// 5. label
node = (*region).FindValue("label");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- region"
<< name << "has no label";
std::string label;
*node >> label;
Region *r = addRegionFromBundle(name, nodeType, dimensions, fullPath, label);
setPhases_(r, phases);
}
const YAML::Node *links = doc.FindValue("Links");
if (links == NULL)
NTA_THROW << "Invalid network structure file -- no links";
if (links->Type() != YAML::NodeType::Sequence)
NTA_THROW << "Invalid network structure file -- links element is not a list";
for (YAML::Iterator link = links->begin(); link != links->end(); link++)
{
// Each link is a map -- extract the 5 values in the map
if ((*link).Type() != YAML::NodeType::Map)
NTA_THROW << "Invalid network structure file -- bad link (not a map)";
if ((*link).size() != 6)
NTA_THROW << "Invalid network structure file -- bad link (wrong size)";
// 1. type
node = (*link).FindValue("type");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- link does not have a type";
std::string linkType;
*node >> linkType;
// 2. params
node = (*link).FindValue("params");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- link does not have params";
std::string params;
*node >> params;
// 3. srcRegion (name)
node = (*link).FindValue("srcRegion");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- link does not have a srcRegion";
std::string srcRegionName;
*node >> srcRegionName;
// 4. srcOutput
node = (*link).FindValue("srcOutput");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- link does not have a srcOutput";
std::string srcOutputName;
*node >> srcOutputName;
// 5. destRegion
node = (*link).FindValue("destRegion");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- link does not have a destRegion";
std::string destRegionName;
*node >> destRegionName;
// 6. destInput
node = (*link).FindValue("destInput");
if (node == NULL)
NTA_THROW << "Invalid network structure file -- link does not have a destInput";
std::string destInputName;
*node >> destInputName;
if (!regions_.contains(srcRegionName))
NTA_THROW << "Invalid network structure file -- link specifies source region '" << srcRegionName << "' but no such region exists";
Region* srcRegion = regions_.getByName(srcRegionName);
if (!regions_.contains(destRegionName))
NTA_THROW << "Invalid network structure file -- link specifies destination region '" << destRegionName << "' but no such region exists";
Region* destRegion = regions_.getByName(destRegionName);
Output* srcOutput = srcRegion->getOutput(srcOutputName);
if (srcOutput == NULL)
NTA_THROW << "Invalid network structure file -- link specifies source output '" << srcOutputName << "' but no such name exists";
Input* destInput = destRegion->getInput(destInputName);
if (destInput == NULL)
NTA_THROW << "Invalid network structure file -- link specifies destination input '" << destInputName << "' but no such name exists";
// Create the link itself
destInput->addLink(linkType, params, srcOutput);
} // links
}
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, shared_ptr< Query> query)
{
assert(schemas.size() == 1);
assert(_parameters.size() == 1);
string arrayName = ((shared_ptr<OperatorParamReference>&)_parameters[0])->getObjectName();
ArrayDesc const& srcDesc = schemas[0];
//Ensure attributes names uniqueness.
ArrayDesc dstDesc;
if (!SystemCatalog::getInstance()->getArrayDesc(arrayName, dstDesc, false))
{
Attributes outAttrs;
map<string, uint64_t> attrsMap;
BOOST_FOREACH(const AttributeDesc &attr, srcDesc.getAttributes())
{
AttributeDesc newAttr;
if (!attrsMap.count(attr.getName()))
{
attrsMap[attr.getName()] = 1;
newAttr = attr;
}
else
{
while (true) {
stringstream ss;
ss << attr.getName() << "_" << ++attrsMap[attr.getName()];
if (attrsMap.count(ss.str()) == 0) {
newAttr = AttributeDesc(attr.getId(), ss.str(), attr.getType(), attr.getFlags(),
attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(),
attr.getDefaultValueExpr());
attrsMap[ss.str()] = 1;
break;
}
}
}
outAttrs.push_back(newAttr);
}
Dimensions outDims;
map<string, uint64_t> dimsMap;
BOOST_FOREACH(const DimensionDesc &dim, srcDesc.getDimensions())
{
DimensionDesc newDim;
if (!dimsMap.count(dim.getBaseName()))
{
dimsMap[dim.getBaseName()] = 1;
newDim = DimensionDesc(dim.getBaseName(),
dim.getStartMin(),
dim.getCurrStart(),
dim.getCurrEnd(),
dim.getEndMax(),
dim.getChunkInterval(),
dim.getChunkOverlap());
}
else
{
while (true) {
stringstream ss;
ss << dim.getBaseName() << "_" << ++dimsMap[dim.getBaseName()];
if (dimsMap.count(ss.str()) == 0) {
newDim = DimensionDesc(ss.str(),
dim.getStartMin(),
dim.getCurrStart(),
dim.getCurrEnd(),
dim.getEndMax(),
dim.getChunkInterval(),
dim.getChunkOverlap());
dimsMap[ss.str()] = 1;
break;
}
}
}
outDims.push_back(newDim);
}
/* Notice that when storing to a non-existant array, we do not propagate the
transience of the source array to to the target ...*/
return ArrayDesc(arrayName, outAttrs, outDims, srcDesc.getFlags() & (~ArrayDesc::TRANSIENT));
}
void DimensionsTest::RunTests()
{
Coordinate zero; // [0];
zero.push_back(0);
Coordinate one_two; // [1,2]
one_two.push_back(1);
one_two.push_back(2);
Coordinate three_four; // [3,4]
three_four.push_back(3);
three_four.push_back(4);
{
// empty dimensions (unspecified)
Dimensions d;
TEST(d.isUnspecified());
TEST(d.isValid());
TEST(!d.isDontcare());
SHOULDFAIL(d.getCount());
SHOULDFAIL(d.getDimension(0));
TESTEQUAL("[unspecified]", d.toString());
SHOULDFAIL(d.getIndex(one_two));
SHOULDFAIL(d.getCount());
SHOULDFAIL(d.getDimension(0));
TESTEQUAL((unsigned int)0, d.getDimensionCount());
}
{
// dontcare dimensions [0]
Dimensions d;
d.push_back(0);
TEST(!d.isUnspecified());
TEST(d.isDontcare());
TEST(d.isValid());
TESTEQUAL("[dontcare]", d.toString());
SHOULDFAIL(d.getIndex(zero));
SHOULDFAIL(d.getCount());
TESTEQUAL((unsigned int)0, d.getDimension(0));
TESTEQUAL((unsigned int)1, d.getDimensionCount());
}
{
// invalid dimensions
Dimensions d;
d.push_back(1);
d.push_back(0);
TEST(!d.isUnspecified());
TEST(!d.isDontcare());
TEST(!d.isValid());
TESTEQUAL("[1 0] (invalid)", d.toString());
SHOULDFAIL(d.getIndex(one_two));
SHOULDFAIL(d.getCount());
TESTEQUAL((unsigned int)1, d.getDimension(0));
TESTEQUAL((unsigned int)0, d.getDimension(1));
SHOULDFAIL(d.getDimension(2));
TESTEQUAL((unsigned int)2, d.getDimensionCount());
}
{
// valid dimensions [2,3]
// two rows, three columns
Dimensions d;
d.push_back(2);
d.push_back(3);
TEST(!d.isUnspecified());
TEST(!d.isDontcare());
TEST(d.isValid());
TESTEQUAL("[2 3]", d.toString());
TESTEQUAL((unsigned int)2, d.getDimension(0));
TESTEQUAL((unsigned int)3, d.getDimension(1));
SHOULDFAIL(d.getDimension(2));
TESTEQUAL((unsigned int)6, d.getCount());
TESTEQUAL((unsigned int)5, d.getIndex(one_two));
TESTEQUAL((unsigned int)2, d.getDimensionCount());
}
{
//check a two dimensional matrix for proper x-major ordering
std::vector<size_t> x;
x.push_back(4);
x.push_back(5);
Dimensions d(x);
size_t testDim1 = 4;
size_t testDim2 = 5;
for(size_t i = 0; i < testDim1; i++)
{
for(size_t j = 0; j < testDim2; j++)
{
Coordinate testCoordinate;
testCoordinate.push_back(i);
testCoordinate.push_back(j);
TESTEQUAL(i+j*testDim1, d.getIndex(testCoordinate));
TESTEQUAL(vecToString(testCoordinate),
vecToString(d.getCoordinate(i+j*testDim1)));
}
}
}
{
//check a three dimensional matrix for proper x-major ordering
std::vector<size_t> x;
x.push_back(3);
x.push_back(4);
x.push_back(5);
Dimensions d(x);
size_t testDim1 = 3;
size_t testDim2 = 4;
size_t testDim3 = 5;
for(size_t i = 0; i < testDim1; i++)
{
for(size_t j = 0; j < testDim2; j++)
{
for(size_t k = 0; k < testDim3; k++)
{
Coordinate testCoordinate;
testCoordinate.push_back(i);
testCoordinate.push_back(j);
testCoordinate.push_back(k);
TESTEQUAL(i +
j*testDim1 +
k*testDim1*testDim2, d.getIndex(testCoordinate));
TESTEQUAL(vecToString(testCoordinate),
vecToString(d.getCoordinate(i +
j*testDim1 +
k*testDim1*testDim2)));
}
}
}
}
{
// alternate constructor
std::vector<size_t> x;
x.push_back(2);
x.push_back(5);
Dimensions d(x);
TEST(!d.isUnspecified());
TEST(!d.isDontcare());
TEST(d.isValid());
TESTEQUAL((unsigned int)2, d.getDimension(0));
TESTEQUAL((unsigned int)5, d.getDimension(1));
SHOULDFAIL(d.getDimension(2));
TESTEQUAL((unsigned int)2, d.getDimensionCount());
}
} // RunTests()
TEST(WatcherTest, SampleNetwork)
{
//generate sample network
Network n;
n.addRegion("level1", "TestNode", "");
n.addRegion("level2", "TestNode", "");
n.addRegion("level3", "TestNode", "");
Dimensions d;
d.push_back(8);
d.push_back(4);
n.getRegions().getByName("level1")->setDimensions(d);
n.link("level1", "level2", "TestFanIn2", "");
n.link("level2", "level3", "TestFanIn2", "");
n.initialize();
//erase any previous contents of testfile
OFStream o("testfile");
o.close();
//test creation
Watcher w("testfile");
//test uint32Params
unsigned int id1 = w.watchParam("level1", "uint32Param");
ASSERT_EQ(id1, (unsigned int)1);
//test uint64Params
unsigned int id2 = w.watchParam("level1", "uint64Param");
ASSERT_EQ(id2, (unsigned int)2);
//test int32Params
w.watchParam("level1", "int32Param");
//test int64Params
w.watchParam("level1", "int64Param");
//test real32Params
w.watchParam("level1", "real32Param");
//test real64Params
w.watchParam("level1", "real64Param");
//test stringParams
w.watchParam("level1", "stringParam");
//test unclonedParams
w.watchParam("level1", "unclonedParam", 0);
w.watchParam("level1", "unclonedParam", 1);
//test attachToNetwork()
w.attachToNetwork(n);
//test two simultaneous Watchers on the same network with different files
Watcher* w2 = new Watcher("testfile2");
//test int64ArrayParam
w2->watchParam("level1", "int64ArrayParam");
//test real32ArrayParam
w2->watchParam("level1", "real32ArrayParam");
//test output
w2->watchOutput("level1", "bottomUpOut");
//test int64ArrayParam, sparse = false
w2->watchParam("level1", "int64ArrayParam", -1, false);
w2->attachToNetwork(n);
//set one of the uncloned parameters to 1 instead of 0
//n.getRegions().getByName("level1")->getNodeAtIndex(1).setParameterUInt32("unclonedParam", (UInt32)1);
//n.run(3);
//see if Watcher notices change in parameter values after 3 iterations
n.getRegions().getByName("level1")->setParameterUInt64("uint64Param", (UInt64)66);
n.run(3);
//test flushFile() - this should produce output
w.flushFile();
//test closeFile()
w.closeFile();
//test to make sure data is flushed when Watcher is deleted
delete w2;
}
