ArrayDesc inferSchema(std::vector<ArrayDesc> schemas, boost::shared_ptr<Query> query) { Attributes atts; atts.push_back(AttributeDesc(0, "attribute_name", TID_STRING, 0, 0)); atts.push_back(AttributeDesc(1, "min", TID_STRING, 0, 0)); atts.push_back(AttributeDesc(2, "max", TID_STRING, 0, 0)); atts.push_back(AttributeDesc(3, "distinct_count", TID_UINT64, 0, 0)); atts.push_back(AttributeDesc(4, "non_null_count", TID_UINT64, 0, 0)); const AttributeDesc *emptyIndicator = schemas[0].getEmptyBitmapAttribute(); set<string> a_s; for (size_t i = 0; i < _parameters.size(); i++) { string attName = ((boost::shared_ptr<OperatorParamReference>&)_parameters[i])->getObjectName(); if (emptyIndicator && emptyIndicator->getName() == attName) continue; a_s.insert(attName); } size_t attsCount = (a_s.size() == 0 ? (emptyIndicator ? schemas[0].getAttributes().size() - 1 : schemas[0].getAttributes().size()) : a_s.size()) - 1; Dimensions dims; dims.push_back(DimensionDesc("attribute_number", 0, attsCount, ANALYZE_CHUNK_SIZE, 0)); return ArrayDesc(schemas[0].getName() + "_analyze", atts, dims); }
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query) { Attributes outputAttrs; outputAttrs.push_back(AttributeDesc(0, "dummy", TID_DOUBLE, AttributeDesc::IS_NULLABLE, 0)); Dimensions outputDims; outputDims.push_back(DimensionDesc("i",0,0,1,0)); return ArrayDesc("test_cache", outputAttrs, outputDims); }
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query) { assert(schemas.size() == 2); ArrayDesc const& patternDesc = schemas[0]; ArrayDesc const& catalogDesc = schemas[1]; Attributes const& catalogAttributes = catalogDesc.getAttributes(true); Dimensions const& catalogDimensions = catalogDesc.getDimensions(); Attributes const& patternAttributes = patternDesc.getAttributes(true); Dimensions resultDimensions = patternDesc.getDimensions(); size_t totalAttributes = catalogAttributes.size() + patternAttributes.size() + 1 + catalogDimensions.size(); Attributes matchAttributes(totalAttributes); if (catalogDimensions.size() != resultDimensions.size()) { stringstream left, right; printDimNames(left, resultDimensions); printDimNames(right, catalogDimensions); throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_DIMENSION_COUNT_MISMATCH) << "match" << left.str() << right.str(); } for (size_t i = 0, n = catalogDimensions.size(); i < n; i++) { if (!(catalogDimensions[i].getStartMin() == resultDimensions[i].getStartMin() && catalogDimensions[i].getChunkInterval() == resultDimensions[i].getChunkInterval() && catalogDimensions[i].getChunkOverlap() == resultDimensions[i].getChunkOverlap())) { // XXX To do: implement requiresRepart() method, remove interval/overlap checks // above, use SCIDB_LE_START_INDEX_MISMATCH here. throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_ARRAYS_NOT_CONFORMANT); } } size_t j = 0; for (size_t i = 0, n = patternAttributes.size(); i < n; i++, j++) { AttributeDesc const& attr = patternAttributes[i]; matchAttributes[j] = AttributeDesc(j, attr.getName(), attr.getType(), attr.getFlags(), attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(), attr.getDefaultValueExpr()); } for (size_t i = 0, n = catalogAttributes.size(); i < n; i++, j++) { AttributeDesc const& attr = catalogAttributes[i]; matchAttributes[j] = AttributeDesc(j, "match_" + attr.getName(), attr.getType(), attr.getFlags(), attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(), attr.getDefaultValueExpr()); } for (size_t i = 0, n = catalogDimensions.size(); i < n; i++, j++) { matchAttributes[j] = AttributeDesc(j, "match_" + catalogDimensions[i].getBaseName(), TID_INT64, 0, 0); } matchAttributes[j] = AttributeDesc(j, DEFAULT_EMPTY_TAG_ATTRIBUTE_NAME, TID_INDICATOR, AttributeDesc::IS_EMPTY_INDICATOR, 0); int64_t maxCollisions = evaluate(((boost::shared_ptr<OperatorParamLogicalExpression>&)_parameters[1])->getExpression(), query, TID_INT64).getInt64(); if (maxCollisions <= 0 || (int32_t)maxCollisions != maxCollisions) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_OPERATOR_ARGUMENT2) << "positive"; } resultDimensions.push_back(DimensionDesc("collision", 0, 0, maxCollisions-1, maxCollisions-1, (uint32_t)maxCollisions, 0)); return ArrayDesc("match", matchAttributes, resultDimensions); }
TEST_F(DimensionsTest, DontCareDimensions) { // dontcare dimensions [0] Dimensions d; d.push_back(0); ASSERT_TRUE(!d.isUnspecified()); ASSERT_TRUE(d.isDontcare()); ASSERT_TRUE(d.isValid()); EXPECT_STREQ("[dontcare]", d.toString().c_str()); ASSERT_ANY_THROW(d.getIndex(zero)); ASSERT_ANY_THROW(d.getCount()); ASSERT_EQ((unsigned int)0, d.getDimension(0)); ASSERT_EQ((unsigned int)1, d.getDimensionCount()); }
TEST_F(DimensionsTest, InvalidDimensions) { // invalid dimensions Dimensions d; d.push_back(1); d.push_back(0); ASSERT_TRUE(!d.isUnspecified()); ASSERT_TRUE(!d.isDontcare()); ASSERT_TRUE(!d.isValid()); EXPECT_STREQ("[1 0] (invalid)", d.toString().c_str()); ASSERT_ANY_THROW(d.getIndex(one_two)); ASSERT_ANY_THROW(d.getCount()); ASSERT_EQ((unsigned int)1, d.getDimension(0)); ASSERT_EQ((unsigned int)0, d.getDimension(1)); ASSERT_ANY_THROW(d.getDimension(2)); ASSERT_EQ((unsigned int)2, d.getDimensionCount()); }
TEST_F(DimensionsTest, ValidDimensions) { // valid dimensions [2,3] // two rows, three columns Dimensions d; d.push_back(2); d.push_back(3); ASSERT_TRUE(!d.isUnspecified()); ASSERT_TRUE(!d.isDontcare()); ASSERT_TRUE(d.isValid()); EXPECT_STREQ("[2 3]", d.toString().c_str()); ASSERT_EQ((unsigned int)2, d.getDimension(0)); ASSERT_EQ((unsigned int)3, d.getDimension(1)); ASSERT_ANY_THROW(d.getDimension(2)); ASSERT_EQ((unsigned int)6, d.getCount()); ASSERT_EQ((unsigned int)5, d.getIndex(one_two)); ASSERT_EQ((unsigned int)2, d.getDimensionCount()); }
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query) { assert(schemas.size() == 1); ArrayDesc const& srcDesc = schemas[0]; ArrayDesc dstDesc = ((boost::shared_ptr<OperatorParamSchema>&)_parameters[0])->getSchema(); //Compile a desc of all possible attributes (aggregate calls first) and source dimensions ArrayDesc aggregationDesc (srcDesc.getName(), Attributes(), srcDesc.getDimensions()); vector<string> aggregatedNames; //add aggregate calls first for (size_t i = 1; i < _parameters.size(); i++) { addAggregatedAttribute( (shared_ptr <OperatorParamAggregateCall>&) _parameters[i], srcDesc, aggregationDesc); string aggName = aggregationDesc.getAttributes()[aggregationDesc.getAttributes().size()-1].getName(); bool aggFound = false; BOOST_FOREACH(const AttributeDesc &dstAttr, dstDesc.getAttributes()) { if (dstAttr.getName() == aggName) { aggFound = true; break; } } if (!aggFound) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_ATTRIBUTE_DOESNT_EXIST) << aggName << dstDesc.getName(); } aggregatedNames.push_back(aggName); } //add other attributes BOOST_FOREACH(const AttributeDesc &srcAttr, srcDesc.getAttributes()) { //if there's an attribute with same name as an aggregate call - skip the attribute bool found = false; BOOST_FOREACH(const AttributeDesc &aggAttr, aggregationDesc.getAttributes()) { if( aggAttr.getName() == srcAttr.getName()) { found = true; } } if (!found) { aggregationDesc.addAttribute(AttributeDesc( aggregationDesc.getAttributes().size(), srcAttr.getName(), srcAttr.getType(), srcAttr.getFlags(), srcAttr.getDefaultCompressionMethod(), srcAttr.getAliases(), &srcAttr.getDefaultValue(), srcAttr.getDefaultValueExpr(), srcAttr.getVarSize())); } } //Ensure attributes names uniqueness. if (!dstDesc.getEmptyBitmapAttribute()) throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OP_REDIMENSION_ERROR1); BOOST_FOREACH(const AttributeDesc &dstAttr, dstDesc.getAttributes()) { BOOST_FOREACH(const AttributeDesc &srcAttr, aggregationDesc.getAttributes()) { if (srcAttr.getName() == dstAttr.getName()) { if (srcAttr.getType() != dstAttr.getType()) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_ATTRIBUTE_TYPE) << srcAttr.getName() << srcAttr.getType() << dstAttr.getType(); } if (!dstAttr.isNullable() && srcAttr.isNullable()) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_ATTRIBUTE_FLAGS) << srcAttr.getName(); } goto NextAttr; } } BOOST_FOREACH(const DimensionDesc &srcDim, aggregationDesc.getDimensions()) { if (srcDim.hasNameAndAlias(dstAttr.getName())) { if (dstAttr.getType() != TID_INT64) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_DESTINATION_ATTRIBUTE_TYPE) << dstAttr.getName() << TID_INT64; } if (dstAttr.getFlags() != 0) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_DESTINATION_ATTRIBUTE_FLAGS) << dstAttr.getName(); } goto NextAttr; } } if (dstAttr.isEmptyIndicator() == false) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_UNEXPECTED_DESTINATION_ATTRIBUTE) << dstAttr.getName(); } NextAttr:; } Dimensions outputDims; size_t nNewDims = 0; BOOST_FOREACH(const DimensionDesc &dstDim, dstDesc.getDimensions()) { if (dstDim.getChunkOverlap() > dstDim.getChunkInterval()) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OVERLAP_CANT_BE_LARGER_CHUNK); } BOOST_FOREACH(const AttributeDesc &srcAttr, aggregationDesc.getAttributes()) { if (dstDim.hasNameAndAlias(srcAttr.getName())) { for (size_t i = 0; i< aggregatedNames.size(); i++) { if (srcAttr.getName() == aggregatedNames[i]) throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OP_REDIMENSION_ERROR2); } if ( !IS_INTEGRAL(srcAttr.getType()) || srcAttr.getType() == TID_UINT64 ) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_SOURCE_ATTRIBUTE_TYPE) << srcAttr.getName() << TID_INT64; } outputDims.push_back(dstDim); goto NextDim; } } BOOST_FOREACH(const DimensionDesc &srcDim, aggregationDesc.getDimensions()) { if (srcDim.hasNameAndAlias(dstDim.getBaseName())) { DimensionDesc outputDim = dstDim; outputDims.push_back(outputDim); goto NextDim; } } //one synthetic dimension allowed if (nNewDims++ != 0 || !aggregatedNames.empty() ) { throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_UNEXPECTED_DESTINATION_DIMENSION) << dstDim.getBaseName(); } outputDims.push_back(dstDim); NextDim:; } return ArrayDesc(srcDesc.getName(), dstDesc.getAttributes(), outputDims, dstDesc.getFlags()); }
void Link::initialize(size_t destinationOffset) { // Make sure all information is specified and // consistent. Unless there is a NuPIC implementation // error, all these checks are guaranteed to pass // because of the way the network is constructed // and initialized. // Make sure we have been attached to a real network NTA_CHECK(src_ != nullptr); NTA_CHECK(dest_ != nullptr); // Confirm that our dimensions are consistent with the // dimensions of the regions we're connecting. const Dimensions& srcD = getSrcDimensions(); const Dimensions& destD = getDestDimensions(); NTA_CHECK(! srcD.isUnspecified()); NTA_CHECK(! destD.isUnspecified()); Dimensions oneD; oneD.push_back(1); if(src_->isRegionLevel()) { Dimensions d; for(size_t i = 0; i < src_->getRegion().getDimensions().size(); i++) { d.push_back(1); } NTA_CHECK(srcD.isDontcare() || srcD == d); } else if(src_->getRegion().getDimensions() == oneD) { Dimensions d; for(size_t i = 0; i < srcD.size(); i++) { d.push_back(1); } NTA_CHECK(srcD.isDontcare() || srcD == d); } else { NTA_CHECK(srcD.isDontcare() || srcD == src_->getRegion().getDimensions()); } if(dest_->isRegionLevel()) { Dimensions d; for(size_t i = 0; i < dest_->getRegion().getDimensions().size(); i++) { d.push_back(1); } NTA_CHECK(destD.isDontcare() || destD.isOnes()); } else if(dest_->getRegion().getDimensions() == oneD) { Dimensions d; for(size_t i = 0; i < destD.size(); i++) { d.push_back(1); } NTA_CHECK(destD.isDontcare() || destD == d); } else { NTA_CHECK(destD.isDontcare() || destD == dest_->getRegion().getDimensions()); } destOffset_ = destinationOffset; impl_->initialize(); initialized_ = true; }
void RegionTest::testWithNodeType(const std::string& nodeType) { Region *rP = NULL; SHOULDFAIL ( rP = new Region("r1", "nosuchnode", "") ); Region r("r1", nodeType, ""); TEST(r.getName() == "r1"); TEST(r.getType() == nodeType); Dimensions d = r.getDimensions(); TEST(d.isUnspecified()); d.clear(); d.push_back(3); d.push_back(2); r.setDimensions(d); Dimensions d2 = r.getDimensions(); TEST(d2.size() == 2); TEST(d2[0] == 3); TEST(d2[1] == 2); TEST(d2.getCount() == 6); // Parameter testing { { Int32 val = -(1 << 24); TESTEQUAL((Int32)32, r.getParameterInt32("int32Param")); r.setParameterInt32("int32Param", val); TESTEQUAL(val, r.getParameterInt32("int32Param")); } { UInt32 val = 1 << 24; TESTEQUAL((UInt32)33, r.getParameterUInt32("uint32Param")); r.setParameterUInt32("uint32Param", val); TESTEQUAL(val, r.getParameterUInt32("uint32Param")); } { Int64 val = -((Int64)1 << 44); TESTEQUAL((Int64)64, r.getParameterInt64("int64Param")); r.setParameterInt64("int64Param", val); TESTEQUAL(val, r.getParameterInt64("int64Param")); } { UInt64 val = (UInt64)1 << 45; TESTEQUAL((UInt64)65, r.getParameterUInt64("uint64Param")); r.setParameterUInt64("uint64Param", val); TESTEQUAL(r.getParameterUInt64("uint64Param"), val); } { Real32 val = 23456.7; TESTEQUAL((Real32)32.1, r.getParameterReal32("real32Param")); r.setParameterReal32("real32Param", val); TESTEQUAL(r.getParameterReal32("real32Param"), val); } { Real64 val = 23456.789; TESTEQUAL((Real64)64.1, r.getParameterReal64("real64Param")); r.setParameterReal64("real64Param", val); TESTEQUAL(r.getParameterReal64("real64Param"), val); } { Array a(NTA_BasicType_Int64); r.getParameterArray("int64ArrayParam", a); // check default values TESTEQUAL((size_t)4, a.getCount()); Int64 *buf = (Int64*) a.getBuffer(); TEST(buf != NULL); for (UInt64 i = 0; i < 4; i++) TESTEQUAL((Int64)(i*64), buf[i]); // set our own value buf[0] = 100; r.setParameterArray("int64ArrayParam", a); // make sure we retrieve the value just set buf[0] = 0; r.getParameterArray("int64ArrayParam", a); TEST(buf == a.getBuffer()); TESTEQUAL((Int64)100, buf[0]); } { std::string s = r.getParameterString("stringParam"); TESTEQUAL("nodespec value", s); s = "new value"; r.setParameterString("stringParam", s); s = r.getParameterString("stringParam"); TESTEQUAL("new value", s); } } }
int main(int argc, const char * argv[]) { // Create network Network net = Network(); // Add VectorFileSensor region to network Region* region = net.addRegion("region", "VectorFileSensor", "{activeOutputCount: 1}"); // Set region dimensions Dimensions dims; dims.push_back(1); std::cout << "Setting region dimensions" << dims.toString() << std::endl; region->setDimensions(dims); // Load data std::string path = Path::makeAbsolute("../../src/examples/regions/Data.csv"); std::cout << "Loading data from " << path << std::endl; std::vector<std::string> loadFileArgs; loadFileArgs.push_back("loadFile"); loadFileArgs.push_back(path); loadFileArgs.push_back("2"); region->executeCommand(loadFileArgs); // Initialize network std::cout << "Initializing network" << std::endl; net.initialize(); ArrayRef outputArray = region->getOutputData("dataOut"); // Compute std::cout << "Compute" << std::endl; region->compute(); // Get output Real64 *buffer = (Real64*) outputArray.getBuffer(); for (size_t i = 0; i < outputArray.getCount(); i++) { std::cout << " " << i << " " << buffer[i] << "" << std::endl; } // Serialize Network net2; { std::stringstream ss; net.write(ss); net2.read(ss); } net2.initialize(); Region* region2 = net2.getRegions().getByName("region"); region2->executeCommand(loadFileArgs); ArrayRef outputArray2 = region2->getOutputData("dataOut"); Real64 *buffer2 = (Real64*)outputArray2.getBuffer(); net.run(1); net2.run(1); NTA_ASSERT(outputArray2.getCount() == outputArray.getCount()); for (size_t i = 0; i < outputArray.getCount(); i++) { std::cout << " " << i << " " << buffer[i] << " " << buffer2[i] << std::endl; } return 0; }
int realmain(bool leakTest) { // verbose == true turns on extra output that is useful for // debugging the test (e.g. when the TestNode compute() // algorithm changes) std::cout << "Creating network..." << std::endl; Network n; std::cout << "Region count is " << n.getRegions().getCount() << "" << std::endl; std::cout << "Adding a PyNode region..." << std::endl; Network::registerPyRegion("nupic.regions.TestNode", "TestNode"); Region* level2 = n.addRegion("level2", "py.TestNode", "{int32Param: 444}"); std::cout << "Region count is " << n.getRegions().getCount() << "" << std::endl; std::cout << "Node type: " << level2->getType() << "" << std::endl; std::cout << "Nodespec is:\n" << level2->getSpec()->toString() << "" << std::endl; Real64 rval; std::string int64Param("int64Param"); std::string real64Param("real64Param"); // get the value of intArrayParam after the setParameter call. // --- Test getParameterReal64 of a PyNode rval = level2->getParameterReal64("real64Param"); NTA_CHECK(rval == 64.1); std::cout << "level2 getParameterReal64() returned: " << rval << std::endl; // --- Test setParameterReal64 of a PyNode level2->setParameterReal64("real64Param", 77.7); rval = level2->getParameterReal64("real64Param"); NTA_CHECK(rval == 77.7); // should fail because network has not been initialized SHOULDFAIL(n.run(1)); // should fail because network can't be initialized SHOULDFAIL (n.initialize() ); std::cout << "Setting dimensions of level1..." << std::endl; Dimensions d; d.push_back(4); d.push_back(4); std::cout << "Setting dimensions of level2..." << std::endl; level2->setDimensions(d); std::cout << "Initializing again..." << std::endl; n.initialize(); testExceptionBug(); testPynodeInputOutputAccess(level2); testPynodeArrayParameters(level2); testPynodeLinking(); if (!leakTest) { //testNuPIC1x(); //testPynode1xLinking(); } std::cout << "Done -- all tests passed" << std::endl; return 0; }
void Network::loadFromBundle(const std::string& name) { if (! StringUtils::endsWith(name, ".nta")) NTA_THROW << "loadFromBundle: bundle extension must be \".nta\""; std::string fullPath = Path::normalize(Path::makeAbsolute(name)); if (! Path::exists(fullPath)) NTA_THROW << "Path " << fullPath << " does not exist"; std::string networkStructureFilename = Path::join(fullPath, "network.yaml"); std::ifstream f(networkStructureFilename.c_str()); YAML::Parser parser(f); YAML::Node doc; bool success = parser.GetNextDocument(doc); if (!success) NTA_THROW << "Unable to find YAML document in network structure file " << networkStructureFilename; if (doc.Type() != YAML::NodeType::Map) NTA_THROW << "Invalid network structure file -- does not contain a map"; // Should contain Version, Regions, Links if (doc.size() != 3) NTA_THROW << "Invalid network structure file -- contains " << doc.size() << " elements"; // Extra version const YAML::Node *node = doc.FindValue("Version"); if (node == NULL) NTA_THROW << "Invalid network structure file -- no version"; int version; *node >> version; if (version != 2) NTA_THROW << "Invalid network structure file -- only version 2 supported"; // Regions const YAML::Node *regions = doc.FindValue("Regions"); if (regions == NULL) NTA_THROW << "Invalid network structure file -- no regions"; if (regions->Type() != YAML::NodeType::Sequence) NTA_THROW << "Invalid network structure file -- regions element is not a list"; for (YAML::Iterator region = regions->begin(); region != regions->end(); region++) { // Each region is a map -- extract the 5 values in the map if ((*region).Type() != YAML::NodeType::Map) NTA_THROW << "Invalid network structure file -- bad region (not a map)"; if ((*region).size() != 5) NTA_THROW << "Invalid network structure file -- bad region (wrong size)"; // 1. name node = (*region).FindValue("name"); if (node == NULL) NTA_THROW << "Invalid network structure file -- region has no name"; std::string name; *node >> name; // 2. nodeType node = (*region).FindValue("nodeType"); if (node == NULL) NTA_THROW << "Invalid network structure file -- region " << name << " has no node type"; std::string nodeType; *node >> nodeType; // 3. dimensions node = (*region).FindValue("dimensions"); if (node == NULL) NTA_THROW << "Invalid network structure file -- region " << name << " has no dimensions"; if ((*node).Type() != YAML::NodeType::Sequence) NTA_THROW << "Invalid network structure file -- region " << name << " dimensions specified incorrectly"; Dimensions dimensions; for (YAML::Iterator valiter = (*node).begin(); valiter != (*node).end(); valiter++) { size_t val; (*valiter) >> val; dimensions.push_back(val); } // 4. phases node = (*region).FindValue("phases"); if (node == NULL) NTA_THROW << "Invalid network structure file -- region" << name << "has no phases"; if ((*node).Type() != YAML::NodeType::Sequence) NTA_THROW << "Invalid network structure file -- region " << name << " phases specified incorrectly"; std::set<UInt32> phases; for (YAML::Iterator valiter = (*node).begin(); valiter != (*node).end(); valiter++) { UInt32 val; (*valiter) >> val; phases.insert(val); } // 5. label node = (*region).FindValue("label"); if (node == NULL) NTA_THROW << "Invalid network structure file -- region" << name << "has no label"; std::string label; *node >> label; Region *r = addRegionFromBundle(name, nodeType, dimensions, fullPath, label); setPhases_(r, phases); } const YAML::Node *links = doc.FindValue("Links"); if (links == NULL) NTA_THROW << "Invalid network structure file -- no links"; if (links->Type() != YAML::NodeType::Sequence) NTA_THROW << "Invalid network structure file -- links element is not a list"; for (YAML::Iterator link = links->begin(); link != links->end(); link++) { // Each link is a map -- extract the 5 values in the map if ((*link).Type() != YAML::NodeType::Map) NTA_THROW << "Invalid network structure file -- bad link (not a map)"; if ((*link).size() != 6) NTA_THROW << "Invalid network structure file -- bad link (wrong size)"; // 1. type node = (*link).FindValue("type"); if (node == NULL) NTA_THROW << "Invalid network structure file -- link does not have a type"; std::string linkType; *node >> linkType; // 2. params node = (*link).FindValue("params"); if (node == NULL) NTA_THROW << "Invalid network structure file -- link does not have params"; std::string params; *node >> params; // 3. srcRegion (name) node = (*link).FindValue("srcRegion"); if (node == NULL) NTA_THROW << "Invalid network structure file -- link does not have a srcRegion"; std::string srcRegionName; *node >> srcRegionName; // 4. srcOutput node = (*link).FindValue("srcOutput"); if (node == NULL) NTA_THROW << "Invalid network structure file -- link does not have a srcOutput"; std::string srcOutputName; *node >> srcOutputName; // 5. destRegion node = (*link).FindValue("destRegion"); if (node == NULL) NTA_THROW << "Invalid network structure file -- link does not have a destRegion"; std::string destRegionName; *node >> destRegionName; // 6. destInput node = (*link).FindValue("destInput"); if (node == NULL) NTA_THROW << "Invalid network structure file -- link does not have a destInput"; std::string destInputName; *node >> destInputName; if (!regions_.contains(srcRegionName)) NTA_THROW << "Invalid network structure file -- link specifies source region '" << srcRegionName << "' but no such region exists"; Region* srcRegion = regions_.getByName(srcRegionName); if (!regions_.contains(destRegionName)) NTA_THROW << "Invalid network structure file -- link specifies destination region '" << destRegionName << "' but no such region exists"; Region* destRegion = regions_.getByName(destRegionName); Output* srcOutput = srcRegion->getOutput(srcOutputName); if (srcOutput == NULL) NTA_THROW << "Invalid network structure file -- link specifies source output '" << srcOutputName << "' but no such name exists"; Input* destInput = destRegion->getInput(destInputName); if (destInput == NULL) NTA_THROW << "Invalid network structure file -- link specifies destination input '" << destInputName << "' but no such name exists"; // Create the link itself destInput->addLink(linkType, params, srcOutput); } // links }
ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, shared_ptr< Query> query) { assert(schemas.size() == 1); assert(_parameters.size() == 1); string arrayName = ((shared_ptr<OperatorParamReference>&)_parameters[0])->getObjectName(); ArrayDesc const& srcDesc = schemas[0]; //Ensure attributes names uniqueness. ArrayDesc dstDesc; if (!SystemCatalog::getInstance()->getArrayDesc(arrayName, dstDesc, false)) { Attributes outAttrs; map<string, uint64_t> attrsMap; BOOST_FOREACH(const AttributeDesc &attr, srcDesc.getAttributes()) { AttributeDesc newAttr; if (!attrsMap.count(attr.getName())) { attrsMap[attr.getName()] = 1; newAttr = attr; } else { while (true) { stringstream ss; ss << attr.getName() << "_" << ++attrsMap[attr.getName()]; if (attrsMap.count(ss.str()) == 0) { newAttr = AttributeDesc(attr.getId(), ss.str(), attr.getType(), attr.getFlags(), attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(), attr.getDefaultValueExpr()); attrsMap[ss.str()] = 1; break; } } } outAttrs.push_back(newAttr); } Dimensions outDims; map<string, uint64_t> dimsMap; BOOST_FOREACH(const DimensionDesc &dim, srcDesc.getDimensions()) { DimensionDesc newDim; if (!dimsMap.count(dim.getBaseName())) { dimsMap[dim.getBaseName()] = 1; newDim = DimensionDesc(dim.getBaseName(), dim.getStartMin(), dim.getCurrStart(), dim.getCurrEnd(), dim.getEndMax(), dim.getChunkInterval(), dim.getChunkOverlap()); } else { while (true) { stringstream ss; ss << dim.getBaseName() << "_" << ++dimsMap[dim.getBaseName()]; if (dimsMap.count(ss.str()) == 0) { newDim = DimensionDesc(ss.str(), dim.getStartMin(), dim.getCurrStart(), dim.getCurrEnd(), dim.getEndMax(), dim.getChunkInterval(), dim.getChunkOverlap()); dimsMap[ss.str()] = 1; break; } } } outDims.push_back(newDim); } /* Notice that when storing to a non-existant array, we do not propagate the transience of the source array to to the target ...*/ return ArrayDesc(arrayName, outAttrs, outDims, srcDesc.getFlags() & (~ArrayDesc::TRANSIENT)); }
void DimensionsTest::RunTests() { Coordinate zero; // [0]; zero.push_back(0); Coordinate one_two; // [1,2] one_two.push_back(1); one_two.push_back(2); Coordinate three_four; // [3,4] three_four.push_back(3); three_four.push_back(4); { // empty dimensions (unspecified) Dimensions d; TEST(d.isUnspecified()); TEST(d.isValid()); TEST(!d.isDontcare()); SHOULDFAIL(d.getCount()); SHOULDFAIL(d.getDimension(0)); TESTEQUAL("[unspecified]", d.toString()); SHOULDFAIL(d.getIndex(one_two)); SHOULDFAIL(d.getCount()); SHOULDFAIL(d.getDimension(0)); TESTEQUAL((unsigned int)0, d.getDimensionCount()); } { // dontcare dimensions [0] Dimensions d; d.push_back(0); TEST(!d.isUnspecified()); TEST(d.isDontcare()); TEST(d.isValid()); TESTEQUAL("[dontcare]", d.toString()); SHOULDFAIL(d.getIndex(zero)); SHOULDFAIL(d.getCount()); TESTEQUAL((unsigned int)0, d.getDimension(0)); TESTEQUAL((unsigned int)1, d.getDimensionCount()); } { // invalid dimensions Dimensions d; d.push_back(1); d.push_back(0); TEST(!d.isUnspecified()); TEST(!d.isDontcare()); TEST(!d.isValid()); TESTEQUAL("[1 0] (invalid)", d.toString()); SHOULDFAIL(d.getIndex(one_two)); SHOULDFAIL(d.getCount()); TESTEQUAL((unsigned int)1, d.getDimension(0)); TESTEQUAL((unsigned int)0, d.getDimension(1)); SHOULDFAIL(d.getDimension(2)); TESTEQUAL((unsigned int)2, d.getDimensionCount()); } { // valid dimensions [2,3] // two rows, three columns Dimensions d; d.push_back(2); d.push_back(3); TEST(!d.isUnspecified()); TEST(!d.isDontcare()); TEST(d.isValid()); TESTEQUAL("[2 3]", d.toString()); TESTEQUAL((unsigned int)2, d.getDimension(0)); TESTEQUAL((unsigned int)3, d.getDimension(1)); SHOULDFAIL(d.getDimension(2)); TESTEQUAL((unsigned int)6, d.getCount()); TESTEQUAL((unsigned int)5, d.getIndex(one_two)); TESTEQUAL((unsigned int)2, d.getDimensionCount()); } { //check a two dimensional matrix for proper x-major ordering std::vector<size_t> x; x.push_back(4); x.push_back(5); Dimensions d(x); size_t testDim1 = 4; size_t testDim2 = 5; for(size_t i = 0; i < testDim1; i++) { for(size_t j = 0; j < testDim2; j++) { Coordinate testCoordinate; testCoordinate.push_back(i); testCoordinate.push_back(j); TESTEQUAL(i+j*testDim1, d.getIndex(testCoordinate)); TESTEQUAL(vecToString(testCoordinate), vecToString(d.getCoordinate(i+j*testDim1))); } } } { //check a three dimensional matrix for proper x-major ordering std::vector<size_t> x; x.push_back(3); x.push_back(4); x.push_back(5); Dimensions d(x); size_t testDim1 = 3; size_t testDim2 = 4; size_t testDim3 = 5; for(size_t i = 0; i < testDim1; i++) { for(size_t j = 0; j < testDim2; j++) { for(size_t k = 0; k < testDim3; k++) { Coordinate testCoordinate; testCoordinate.push_back(i); testCoordinate.push_back(j); testCoordinate.push_back(k); TESTEQUAL(i + j*testDim1 + k*testDim1*testDim2, d.getIndex(testCoordinate)); TESTEQUAL(vecToString(testCoordinate), vecToString(d.getCoordinate(i + j*testDim1 + k*testDim1*testDim2))); } } } } { // alternate constructor std::vector<size_t> x; x.push_back(2); x.push_back(5); Dimensions d(x); TEST(!d.isUnspecified()); TEST(!d.isDontcare()); TEST(d.isValid()); TESTEQUAL((unsigned int)2, d.getDimension(0)); TESTEQUAL((unsigned int)5, d.getDimension(1)); SHOULDFAIL(d.getDimension(2)); TESTEQUAL((unsigned int)2, d.getDimensionCount()); } } // RunTests()
TEST(WatcherTest, SampleNetwork) { //generate sample network Network n; n.addRegion("level1", "TestNode", ""); n.addRegion("level2", "TestNode", ""); n.addRegion("level3", "TestNode", ""); Dimensions d; d.push_back(8); d.push_back(4); n.getRegions().getByName("level1")->setDimensions(d); n.link("level1", "level2", "TestFanIn2", ""); n.link("level2", "level3", "TestFanIn2", ""); n.initialize(); //erase any previous contents of testfile OFStream o("testfile"); o.close(); //test creation Watcher w("testfile"); //test uint32Params unsigned int id1 = w.watchParam("level1", "uint32Param"); ASSERT_EQ(id1, (unsigned int)1); //test uint64Params unsigned int id2 = w.watchParam("level1", "uint64Param"); ASSERT_EQ(id2, (unsigned int)2); //test int32Params w.watchParam("level1", "int32Param"); //test int64Params w.watchParam("level1", "int64Param"); //test real32Params w.watchParam("level1", "real32Param"); //test real64Params w.watchParam("level1", "real64Param"); //test stringParams w.watchParam("level1", "stringParam"); //test unclonedParams w.watchParam("level1", "unclonedParam", 0); w.watchParam("level1", "unclonedParam", 1); //test attachToNetwork() w.attachToNetwork(n); //test two simultaneous Watchers on the same network with different files Watcher* w2 = new Watcher("testfile2"); //test int64ArrayParam w2->watchParam("level1", "int64ArrayParam"); //test real32ArrayParam w2->watchParam("level1", "real32ArrayParam"); //test output w2->watchOutput("level1", "bottomUpOut"); //test int64ArrayParam, sparse = false w2->watchParam("level1", "int64ArrayParam", -1, false); w2->attachToNetwork(n); //set one of the uncloned parameters to 1 instead of 0 //n.getRegions().getByName("level1")->getNodeAtIndex(1).setParameterUInt32("unclonedParam", (UInt32)1); //n.run(3); //see if Watcher notices change in parameter values after 3 iterations n.getRegions().getByName("level1")->setParameterUInt64("uint64Param", (UInt64)66); n.run(3); //test flushFile() - this should produce output w.flushFile(); //test closeFile() w.closeFile(); //test to make sure data is flushed when Watcher is deleted delete w2; }