ArrayDesc inferSchema(std::vector<ArrayDesc> schemas, boost::shared_ptr<Query> query)
    {
        Attributes atts;
        atts.push_back(AttributeDesc(0, "attribute_name", TID_STRING, 0, 0));
        atts.push_back(AttributeDesc(1, "min", TID_STRING, 0, 0));
        atts.push_back(AttributeDesc(2, "max", TID_STRING, 0, 0));
        atts.push_back(AttributeDesc(3, "distinct_count", TID_UINT64, 0, 0));
        atts.push_back(AttributeDesc(4, "non_null_count", TID_UINT64, 0, 0));

        const AttributeDesc *emptyIndicator = schemas[0].getEmptyBitmapAttribute();

        set<string> a_s;
        for (size_t i = 0; i < _parameters.size(); i++)
        {
            string attName = ((boost::shared_ptr<OperatorParamReference>&)_parameters[i])->getObjectName();
                        
            if (emptyIndicator && emptyIndicator->getName() == attName)
                continue;

            a_s.insert(attName);
        }

        size_t attsCount = (a_s.size() == 0 ? (emptyIndicator ? schemas[0].getAttributes().size() - 1 : schemas[0].getAttributes().size()) : a_s.size()) - 1;

        Dimensions dims;
        dims.push_back(DimensionDesc("attribute_number", 0, attsCount, ANALYZE_CHUNK_SIZE, 0));

        return ArrayDesc(schemas[0].getName() + "_analyze", atts, dims);
	}
示例#2
0
 ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query)
 {
     Attributes outputAttrs;
     outputAttrs.push_back(AttributeDesc(0, "dummy", TID_DOUBLE, AttributeDesc::IS_NULLABLE, 0));
     Dimensions outputDims;
     outputDims.push_back(DimensionDesc("i",0,0,1,0));
     return ArrayDesc("test_cache", outputAttrs, outputDims);
 }
示例#3
0
    ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query)
    {
        assert(schemas.size() == 2);

        ArrayDesc  const& patternDesc = schemas[0];
        ArrayDesc  const& catalogDesc = schemas[1];
        Attributes const& catalogAttributes = catalogDesc.getAttributes(true);
        Dimensions const& catalogDimensions = catalogDesc.getDimensions();
        Attributes const& patternAttributes = patternDesc.getAttributes(true);
        Dimensions resultDimensions = patternDesc.getDimensions();
        size_t totalAttributes = catalogAttributes.size() + patternAttributes.size() + 1 + catalogDimensions.size();
        Attributes matchAttributes(totalAttributes);

        if (catalogDimensions.size() != resultDimensions.size())
        {
            stringstream left, right;
            printDimNames(left, resultDimensions);
            printDimNames(right, catalogDimensions);
            throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_DIMENSION_COUNT_MISMATCH)
                << "match" << left.str() << right.str();
        }
        for (size_t i = 0, n = catalogDimensions.size(); i < n; i++) {
            if (!(catalogDimensions[i].getStartMin() == resultDimensions[i].getStartMin()
                  && catalogDimensions[i].getChunkInterval() == resultDimensions[i].getChunkInterval()
                  && catalogDimensions[i].getChunkOverlap() == resultDimensions[i].getChunkOverlap()))
            {
                // XXX To do: implement requiresRepart() method, remove interval/overlap checks
                // above, use SCIDB_LE_START_INDEX_MISMATCH here.
                throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_ARRAYS_NOT_CONFORMANT);
            }
        }

        size_t j = 0;
        for (size_t i = 0, n = patternAttributes.size(); i < n; i++, j++) {
            AttributeDesc const& attr = patternAttributes[i];
            matchAttributes[j] = AttributeDesc(j, attr.getName(), attr.getType(), attr.getFlags(),
                                               attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(),
                                               attr.getDefaultValueExpr());
        }
        for (size_t i = 0, n = catalogAttributes.size(); i < n; i++, j++) {
            AttributeDesc const& attr = catalogAttributes[i];
            matchAttributes[j] = AttributeDesc(j, "match_" + attr.getName(), attr.getType(), attr.getFlags(),
                                               attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(),
                                               attr.getDefaultValueExpr());
        }
        for (size_t i = 0, n = catalogDimensions.size(); i < n; i++, j++) {
            matchAttributes[j] = AttributeDesc(j, "match_" + catalogDimensions[i].getBaseName(), TID_INT64, 0, 0);
        }
        matchAttributes[j] = AttributeDesc(j, DEFAULT_EMPTY_TAG_ATTRIBUTE_NAME, TID_INDICATOR, AttributeDesc::IS_EMPTY_INDICATOR, 0);

        int64_t maxCollisions = evaluate(((boost::shared_ptr<OperatorParamLogicalExpression>&)_parameters[1])->getExpression(),
                                          query, TID_INT64).getInt64();
        if (maxCollisions <= 0 || (int32_t)maxCollisions != maxCollisions)  { 
            throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_OPERATOR_ARGUMENT2) << "positive";
        }
        resultDimensions.push_back(DimensionDesc("collision", 0, 0, maxCollisions-1, maxCollisions-1, (uint32_t)maxCollisions, 0));
        return ArrayDesc("match", matchAttributes, resultDimensions);
    }
示例#4
0
TEST_F(DimensionsTest, DontCareDimensions) {
  // dontcare dimensions [0]
  Dimensions d;
  d.push_back(0);
  ASSERT_TRUE(!d.isUnspecified());
  ASSERT_TRUE(d.isDontcare());
  ASSERT_TRUE(d.isValid());
  EXPECT_STREQ("[dontcare]", d.toString().c_str());
  ASSERT_ANY_THROW(d.getIndex(zero));
  ASSERT_ANY_THROW(d.getCount());
  ASSERT_EQ((unsigned int)0, d.getDimension(0));
  ASSERT_EQ((unsigned int)1, d.getDimensionCount());
}
示例#5
0
TEST_F(DimensionsTest, InvalidDimensions) {
  // invalid dimensions
  Dimensions d;
  d.push_back(1);
  d.push_back(0);
  ASSERT_TRUE(!d.isUnspecified());
  ASSERT_TRUE(!d.isDontcare());
  ASSERT_TRUE(!d.isValid());
  EXPECT_STREQ("[1 0] (invalid)", d.toString().c_str());
  ASSERT_ANY_THROW(d.getIndex(one_two));
  ASSERT_ANY_THROW(d.getCount());
  ASSERT_EQ((unsigned int)1, d.getDimension(0));
  ASSERT_EQ((unsigned int)0, d.getDimension(1));
  ASSERT_ANY_THROW(d.getDimension(2));
  ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
示例#6
0
TEST_F(DimensionsTest, ValidDimensions) {
  // valid dimensions [2,3]
  // two rows, three columns
  Dimensions d;
  d.push_back(2);
  d.push_back(3);
  ASSERT_TRUE(!d.isUnspecified());
  ASSERT_TRUE(!d.isDontcare());
  ASSERT_TRUE(d.isValid());
  EXPECT_STREQ("[2 3]", d.toString().c_str());
  ASSERT_EQ((unsigned int)2, d.getDimension(0));
  ASSERT_EQ((unsigned int)3, d.getDimension(1));
  ASSERT_ANY_THROW(d.getDimension(2));
  ASSERT_EQ((unsigned int)6, d.getCount());
  ASSERT_EQ((unsigned int)5, d.getIndex(one_two));
  ASSERT_EQ((unsigned int)2, d.getDimensionCount());
}
    ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, boost::shared_ptr< Query> query)
	{
		assert(schemas.size() == 1);

        ArrayDesc const& srcDesc = schemas[0];
        ArrayDesc dstDesc = ((boost::shared_ptr<OperatorParamSchema>&)_parameters[0])->getSchema();

        //Compile a desc of all possible attributes (aggregate calls first) and source dimensions
        ArrayDesc aggregationDesc (srcDesc.getName(), Attributes(), srcDesc.getDimensions());
        vector<string> aggregatedNames;

        //add aggregate calls first
        for (size_t i = 1; i < _parameters.size(); i++)
        {
            addAggregatedAttribute( (shared_ptr <OperatorParamAggregateCall>&) _parameters[i], srcDesc, aggregationDesc);
            string aggName =  aggregationDesc.getAttributes()[aggregationDesc.getAttributes().size()-1].getName();
            bool aggFound = false;
            BOOST_FOREACH(const AttributeDesc &dstAttr, dstDesc.getAttributes()) { 
                if (dstAttr.getName() == aggName) { 
                    aggFound = true;
                    break;
                }
            }
            if (!aggFound) { 
                throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_ATTRIBUTE_DOESNT_EXIST) << aggName << dstDesc.getName();
            }
            aggregatedNames.push_back(aggName);
        }

        //add other attributes
        BOOST_FOREACH(const AttributeDesc &srcAttr, srcDesc.getAttributes())
        {
            //if there's an attribute with same name as an aggregate call - skip the attribute
            bool found = false;
            BOOST_FOREACH(const AttributeDesc &aggAttr, aggregationDesc.getAttributes())
            {
                if( aggAttr.getName() == srcAttr.getName())
                {
                    found = true;
                }
            }

            if (!found)
            {
                aggregationDesc.addAttribute(AttributeDesc( aggregationDesc.getAttributes().size(),
                                                            srcAttr.getName(),
                                                            srcAttr.getType(),
                                                            srcAttr.getFlags(),
                                                            srcAttr.getDefaultCompressionMethod(),
                                                            srcAttr.getAliases(),
                                                            &srcAttr.getDefaultValue(),
                                                            srcAttr.getDefaultValueExpr(),
                                                            srcAttr.getVarSize()));
            }
        }

        //Ensure attributes names uniqueness.        
        if (!dstDesc.getEmptyBitmapAttribute())
            throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OP_REDIMENSION_ERROR1);
 
        BOOST_FOREACH(const AttributeDesc &dstAttr, dstDesc.getAttributes())
        {
            BOOST_FOREACH(const AttributeDesc &srcAttr, aggregationDesc.getAttributes())
            {
                if (srcAttr.getName() == dstAttr.getName())
                {
                    if (srcAttr.getType() != dstAttr.getType())
                    {
                        throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_ATTRIBUTE_TYPE)
                            << srcAttr.getName() << srcAttr.getType() << dstAttr.getType();
                    }
                    if (!dstAttr.isNullable() && srcAttr.isNullable())
                    {
                        throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_ATTRIBUTE_FLAGS)
                            << srcAttr.getName();
                    }

                    goto NextAttr;
                }
            }
            BOOST_FOREACH(const DimensionDesc &srcDim, aggregationDesc.getDimensions())
            {
                if (srcDim.hasNameAndAlias(dstAttr.getName()))
                {
                    if (dstAttr.getType() != TID_INT64)
                    {
                        throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_DESTINATION_ATTRIBUTE_TYPE)
                            << dstAttr.getName() << TID_INT64;
                    }
                    if (dstAttr.getFlags() != 0)
                    {
                        throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_DESTINATION_ATTRIBUTE_FLAGS)
                            << dstAttr.getName();
                    }

                    goto NextAttr;
                }
            }

            if (dstAttr.isEmptyIndicator() == false)
            {
                throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_UNEXPECTED_DESTINATION_ATTRIBUTE)
                    << dstAttr.getName();
            }
          NextAttr:;
        }
        
        Dimensions outputDims;
        size_t nNewDims = 0;
        BOOST_FOREACH(const DimensionDesc &dstDim, dstDesc.getDimensions())
        {
            if (dstDim.getChunkOverlap() > dstDim.getChunkInterval())
            {
                throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OVERLAP_CANT_BE_LARGER_CHUNK);
            }
            BOOST_FOREACH(const AttributeDesc &srcAttr, aggregationDesc.getAttributes())
            {
                if (dstDim.hasNameAndAlias(srcAttr.getName()))
                {
                    for (size_t i = 0; i< aggregatedNames.size(); i++)
                    {
                        if (srcAttr.getName() == aggregatedNames[i])
                            throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_OP_REDIMENSION_ERROR2);
                    }
                    if ( !IS_INTEGRAL(srcAttr.getType())  || srcAttr.getType() == TID_UINT64 )
                    {
                        throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_WRONG_SOURCE_ATTRIBUTE_TYPE)
                            << srcAttr.getName() << TID_INT64;
                    }
                    outputDims.push_back(dstDim);
                    goto NextDim;
                }
            }
            BOOST_FOREACH(const DimensionDesc &srcDim, aggregationDesc.getDimensions())
            {
                if (srcDim.hasNameAndAlias(dstDim.getBaseName()))
                {
                    DimensionDesc outputDim = dstDim;
                    outputDims.push_back(outputDim);
                    goto NextDim;
                }
            }
            //one synthetic dimension allowed
            if (nNewDims++ != 0 || !aggregatedNames.empty() )
            {
                throw USER_EXCEPTION(SCIDB_SE_INFER_SCHEMA, SCIDB_LE_UNEXPECTED_DESTINATION_DIMENSION) << dstDim.getBaseName();
            }
            outputDims.push_back(dstDim);
            NextDim:;
        }

        return ArrayDesc(srcDesc.getName(), dstDesc.getAttributes(), outputDims, dstDesc.getFlags());
	}
示例#8
0
void Link::initialize(size_t destinationOffset)
{
  // Make sure all information is specified and
  // consistent. Unless there is a NuPIC implementation
  // error, all these checks are guaranteed to pass
  // because of the way the network is constructed
  // and initialized. 
  
  // Make sure we have been attached to a real network
  NTA_CHECK(src_ != nullptr);
  NTA_CHECK(dest_ != nullptr);

  // Confirm that our dimensions are consistent with the 
  // dimensions of the regions we're connecting. 
  const Dimensions& srcD = getSrcDimensions();
  const Dimensions& destD = getDestDimensions();
  NTA_CHECK(! srcD.isUnspecified());
  NTA_CHECK(! destD.isUnspecified());

  Dimensions oneD;
  oneD.push_back(1);

  if(src_->isRegionLevel())
  {
    Dimensions d;
    for(size_t i = 0; i < src_->getRegion().getDimensions().size(); i++)
    {
      d.push_back(1);
    }
    
    NTA_CHECK(srcD.isDontcare() || srcD == d);
  }
  else if(src_->getRegion().getDimensions() == oneD)
  {
    Dimensions d;
    for(size_t i = 0; i < srcD.size(); i++)
    {
      d.push_back(1);
    }
    NTA_CHECK(srcD.isDontcare() || srcD == d);
  }
  else
  {
    NTA_CHECK(srcD.isDontcare() || srcD == src_->getRegion().getDimensions());
  }

  if(dest_->isRegionLevel())
  {
    Dimensions d;
    for(size_t i = 0; i < dest_->getRegion().getDimensions().size(); i++)
    {
      d.push_back(1);
    }
    
    NTA_CHECK(destD.isDontcare() || destD.isOnes());
  }
  else if(dest_->getRegion().getDimensions() == oneD)
  {
    Dimensions d;
    for(size_t i = 0; i < destD.size(); i++)
    {
      d.push_back(1);
    }
    NTA_CHECK(destD.isDontcare() || destD == d);
  }
  else
  {
    NTA_CHECK(destD.isDontcare() || destD == dest_->getRegion().getDimensions());
  }
  
  destOffset_ = destinationOffset;
  impl_->initialize();
  initialized_ = true;
  
}
示例#9
0
void RegionTest::testWithNodeType(const std::string& nodeType)
{

    Region *rP = NULL;

    SHOULDFAIL ( rP = new Region("r1", "nosuchnode", "") );

    Region r("r1", nodeType, "");

    TEST(r.getName() == "r1");

    TEST(r.getType() == nodeType);

    Dimensions d = r.getDimensions();
    TEST(d.isUnspecified());

    d.clear();
    d.push_back(3);
    d.push_back(2);
    r.setDimensions(d);

    Dimensions d2 = r.getDimensions();
    TEST(d2.size() == 2);
    TEST(d2[0] == 3);
    TEST(d2[1] == 2);

    TEST(d2.getCount() == 6);

    // Parameter testing
    {
        {
            Int32 val = -(1 << 24);
            TESTEQUAL((Int32)32, r.getParameterInt32("int32Param"));
            r.setParameterInt32("int32Param", val);
            TESTEQUAL(val, r.getParameterInt32("int32Param"));
        }

        {
            UInt32 val = 1 << 24;
            TESTEQUAL((UInt32)33, r.getParameterUInt32("uint32Param"));
            r.setParameterUInt32("uint32Param", val);
            TESTEQUAL(val, r.getParameterUInt32("uint32Param"));
        }

        {
            Int64 val = -((Int64)1 << 44);
            TESTEQUAL((Int64)64, r.getParameterInt64("int64Param"));
            r.setParameterInt64("int64Param", val);
            TESTEQUAL(val, r.getParameterInt64("int64Param"));
        }

        {
            UInt64 val = (UInt64)1 << 45;
            TESTEQUAL((UInt64)65, r.getParameterUInt64("uint64Param"));
            r.setParameterUInt64("uint64Param", val);
            TESTEQUAL(r.getParameterUInt64("uint64Param"), val);
        }

        {
            Real32 val = 23456.7;
            TESTEQUAL((Real32)32.1, r.getParameterReal32("real32Param"));
            r.setParameterReal32("real32Param", val);
            TESTEQUAL(r.getParameterReal32("real32Param"), val);
        }

        {
            Real64 val = 23456.789;
            TESTEQUAL((Real64)64.1, r.getParameterReal64("real64Param"));
            r.setParameterReal64("real64Param", val);
            TESTEQUAL(r.getParameterReal64("real64Param"), val);
        }

        {
            Array a(NTA_BasicType_Int64);
            r.getParameterArray("int64ArrayParam", a);
            // check default values
            TESTEQUAL((size_t)4, a.getCount());
            Int64 *buf = (Int64*) a.getBuffer();
            TEST(buf != NULL);
            for (UInt64 i = 0; i < 4; i++)
                TESTEQUAL((Int64)(i*64), buf[i]);

            // set our own value
            buf[0] = 100;
            r.setParameterArray("int64ArrayParam", a);
            // make sure we retrieve the value just set
            buf[0] = 0;
            r.getParameterArray("int64ArrayParam", a);
            TEST(buf == a.getBuffer());
            TESTEQUAL((Int64)100, buf[0]);
        }

        {
            std::string s = r.getParameterString("stringParam");
            TESTEQUAL("nodespec value", s);
            s = "new value";
            r.setParameterString("stringParam", s);
            s = r.getParameterString("stringParam");
            TESTEQUAL("new value", s);
        }

    }


}
int main(int argc, const char * argv[])
{
    // Create network
    Network net = Network();

    // Add VectorFileSensor region to network
    Region* region = net.addRegion("region", "VectorFileSensor", "{activeOutputCount: 1}");

    // Set region dimensions
    Dimensions dims;
    dims.push_back(1);

    std::cout << "Setting region dimensions" << dims.toString() << std::endl;

    region->setDimensions(dims);

    // Load data
    std::string path = Path::makeAbsolute("../../src/examples/regions/Data.csv");

    std::cout << "Loading data from " << path << std::endl;

    std::vector<std::string> loadFileArgs;
    loadFileArgs.push_back("loadFile");
    loadFileArgs.push_back(path);
    loadFileArgs.push_back("2");

    region->executeCommand(loadFileArgs);

    // Initialize network
    std::cout << "Initializing network" << std::endl;

    net.initialize();

    ArrayRef outputArray = region->getOutputData("dataOut");

    // Compute
    std::cout << "Compute" << std::endl;

    region->compute();

    // Get output
    Real64 *buffer = (Real64*) outputArray.getBuffer();

    for (size_t i = 0; i < outputArray.getCount(); i++)
    {
        std::cout << "  " << i << "    " << buffer[i] << "" << std::endl;
    }

    // Serialize
    Network net2;
    {
      std::stringstream ss;
      net.write(ss);
      net2.read(ss);
    }
    net2.initialize();

    Region* region2 = net2.getRegions().getByName("region");
    region2->executeCommand(loadFileArgs);
    ArrayRef outputArray2 = region2->getOutputData("dataOut");
    Real64 *buffer2 = (Real64*)outputArray2.getBuffer();

    net.run(1);
    net2.run(1);

    NTA_ASSERT(outputArray2.getCount() == outputArray.getCount());
    for (size_t i = 0; i < outputArray.getCount(); i++)
    {
        std::cout << "  " << i << "    " << buffer[i] << "   " << buffer2[i]
                  << std::endl;
    }

    return 0;
}
示例#11
0
int realmain(bool leakTest)
{
  // verbose == true turns on extra output that is useful for
  // debugging the test (e.g. when the TestNode compute() 
  // algorithm changes)


  std::cout << "Creating network..." << std::endl;
  Network n;
  
  std::cout << "Region count is " << n.getRegions().getCount() << "" << std::endl;

  std::cout << "Adding a PyNode region..." << std::endl;
  Network::registerPyRegion("nupic.regions.TestNode", "TestNode");
  Region* level2 = n.addRegion("level2", "py.TestNode", "{int32Param: 444}");

  std::cout << "Region count is " << n.getRegions().getCount() << "" << std::endl;
  std::cout << "Node type: " << level2->getType() << "" << std::endl;
  std::cout << "Nodespec is:\n"  << level2->getSpec()->toString() << "" << std::endl;
  
  Real64 rval;
  std::string int64Param("int64Param");
  std::string real64Param("real64Param");

  // get the value of intArrayParam after the setParameter call.

  // --- Test getParameterReal64 of a PyNode
  rval = level2->getParameterReal64("real64Param");
  NTA_CHECK(rval == 64.1); 
  std::cout << "level2 getParameterReal64() returned: " << rval << std::endl;

  // --- Test setParameterReal64 of a PyNode
  level2->setParameterReal64("real64Param", 77.7);
  rval = level2->getParameterReal64("real64Param");
  NTA_CHECK(rval == 77.7); 

  // should fail because network has not been initialized
  SHOULDFAIL(n.run(1));

  // should fail because network can't be initialized
  SHOULDFAIL (n.initialize() );

  std::cout << "Setting dimensions of level1..." << std::endl;
  Dimensions d;
  d.push_back(4);
  d.push_back(4);


  std::cout << "Setting dimensions of level2..." << std::endl;
  level2->setDimensions(d);

  std::cout << "Initializing again..." << std::endl;
  n.initialize();

  testExceptionBug();
  testPynodeInputOutputAccess(level2);
  testPynodeArrayParameters(level2);
  testPynodeLinking();
  if (!leakTest)
  {
    //testNuPIC1x();
    //testPynode1xLinking();
  }

  std::cout << "Done -- all tests passed" << std::endl;

  return 0;
}
示例#12
0
void Network::loadFromBundle(const std::string& name)
{
  if (! StringUtils::endsWith(name, ".nta"))
    NTA_THROW << "loadFromBundle: bundle extension must be \".nta\"";

  std::string fullPath = Path::normalize(Path::makeAbsolute(name));

  if (! Path::exists(fullPath))
    NTA_THROW << "Path " << fullPath << " does not exist";

  std::string networkStructureFilename = Path::join(fullPath, "network.yaml");
  std::ifstream f(networkStructureFilename.c_str());
  YAML::Parser parser(f);
  YAML::Node doc;
  bool success = parser.GetNextDocument(doc);
  if (!success)
    NTA_THROW << "Unable to find YAML document in network structure file " 
              << networkStructureFilename;

  if (doc.Type() != YAML::NodeType::Map)
    NTA_THROW << "Invalid network structure file -- does not contain a map";

  // Should contain Version, Regions, Links
  if (doc.size() != 3)
    NTA_THROW << "Invalid network structure file -- contains " 
              << doc.size() << " elements";

  // Extra version
  const YAML::Node *node = doc.FindValue("Version");
  if (node == NULL)
    NTA_THROW << "Invalid network structure file -- no version";
  
  int version;
  *node >> version;
  if (version != 2)
    NTA_THROW << "Invalid network structure file -- only version 2 supported";
  
  // Regions
  const YAML::Node *regions = doc.FindValue("Regions");
  if (regions == NULL)
    NTA_THROW << "Invalid network structure file -- no regions";

  if (regions->Type() != YAML::NodeType::Sequence)
    NTA_THROW << "Invalid network structure file -- regions element is not a list";
  
  for (YAML::Iterator region = regions->begin(); region != regions->end(); region++)
  {
    // Each region is a map -- extract the 5 values in the map
    if ((*region).Type() != YAML::NodeType::Map)
      NTA_THROW << "Invalid network structure file -- bad region (not a map)";
    
    if ((*region).size() != 5)
      NTA_THROW << "Invalid network structure file -- bad region (wrong size)";
    
    // 1. name
    node = (*region).FindValue("name");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- region has no name";
    std::string name;
    *node >> name;

    // 2. nodeType
    node = (*region).FindValue("nodeType");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- region " 
                << name << " has no node type";
    std::string nodeType;
    *node >> nodeType;

    // 3. dimensions
    node = (*region).FindValue("dimensions");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- region "
                << name << " has no dimensions";
    if ((*node).Type() != YAML::NodeType::Sequence)
      NTA_THROW << "Invalid network structure file -- region "
                << name << " dimensions specified incorrectly";
    Dimensions dimensions;
    for (YAML::Iterator valiter = (*node).begin(); valiter != (*node).end(); valiter++)
    {
      size_t val;
      (*valiter) >> val;
      dimensions.push_back(val);
    }

    // 4. phases
    node = (*region).FindValue("phases");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- region"
                << name << "has no phases";
    if ((*node).Type() != YAML::NodeType::Sequence)
      NTA_THROW << "Invalid network structure file -- region "
                << name << " phases specified incorrectly";

    std::set<UInt32> phases;
    for (YAML::Iterator valiter = (*node).begin(); valiter != (*node).end(); valiter++)
    {
      UInt32 val;
      (*valiter) >> val;
      phases.insert(val);
    }
    
    // 5. label
    node = (*region).FindValue("label");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- region"
                << name << "has no label";
    std::string label;
    *node >> label;
    
    Region *r = addRegionFromBundle(name, nodeType, dimensions, fullPath, label);
    setPhases_(r, phases);


  }

  const YAML::Node *links = doc.FindValue("Links");
  if (links == NULL)
    NTA_THROW << "Invalid network structure file -- no links";

  if (links->Type() != YAML::NodeType::Sequence)
    NTA_THROW << "Invalid network structure file -- links element is not a list";

  for (YAML::Iterator link = links->begin(); link != links->end(); link++)
  {
    // Each link is a map -- extract the 5 values in the map
    if ((*link).Type() != YAML::NodeType::Map)
      NTA_THROW << "Invalid network structure file -- bad link (not a map)";
    
    if ((*link).size() != 6)
      NTA_THROW << "Invalid network structure file -- bad link (wrong size)";
    
    // 1. type
    node = (*link).FindValue("type");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- link does not have a type";
    std::string linkType;
    *node >> linkType;

    // 2. params
    node = (*link).FindValue("params");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- link does not have params";
    std::string params;
    *node >> params;

    // 3. srcRegion (name)
    node = (*link).FindValue("srcRegion");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- link does not have a srcRegion";
    std::string srcRegionName;
    *node >> srcRegionName;


    // 4. srcOutput
    node = (*link).FindValue("srcOutput");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- link does not have a srcOutput";
    std::string srcOutputName;
    *node >> srcOutputName;

    // 5. destRegion
    node = (*link).FindValue("destRegion");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- link does not have a destRegion";
    std::string destRegionName;
    *node >> destRegionName;

    // 6. destInput
    node = (*link).FindValue("destInput");
    if (node == NULL)
      NTA_THROW << "Invalid network structure file -- link does not have a destInput";
    std::string destInputName;
    *node >> destInputName;

    if (!regions_.contains(srcRegionName))
      NTA_THROW << "Invalid network structure file -- link specifies source region '" << srcRegionName << "' but no such region exists";
    Region* srcRegion = regions_.getByName(srcRegionName);

    if (!regions_.contains(destRegionName))
      NTA_THROW << "Invalid network structure file -- link specifies destination region '" << destRegionName << "' but no such region exists";
    Region* destRegion = regions_.getByName(destRegionName);

    Output* srcOutput = srcRegion->getOutput(srcOutputName);
    if (srcOutput == NULL)
      NTA_THROW << "Invalid network structure file -- link specifies source output '" << srcOutputName << "' but no such name exists";

    Input* destInput = destRegion->getInput(destInputName);
    if (destInput == NULL)
      NTA_THROW << "Invalid network structure file -- link specifies destination input '" << destInputName << "' but no such name exists";

    // Create the link itself
    destInput->addLink(linkType, params, srcOutput);


  } // links

}
示例#13
0
    ArrayDesc inferSchema(std::vector< ArrayDesc> schemas, shared_ptr< Query> query)
    {
        assert(schemas.size() == 1);
        assert(_parameters.size() == 1);

        string arrayName = ((shared_ptr<OperatorParamReference>&)_parameters[0])->getObjectName();
        ArrayDesc const& srcDesc = schemas[0];

        //Ensure attributes names uniqueness.
        ArrayDesc dstDesc;
        if (!SystemCatalog::getInstance()->getArrayDesc(arrayName, dstDesc, false))
        {
            Attributes outAttrs;
            map<string, uint64_t> attrsMap;
            BOOST_FOREACH(const AttributeDesc &attr, srcDesc.getAttributes())
            {
                AttributeDesc newAttr;
                if (!attrsMap.count(attr.getName()))
                {
                    attrsMap[attr.getName()] = 1;
                    newAttr = attr;
                }
                else
                {
                    while (true) {
                        stringstream ss;
                        ss << attr.getName() << "_" << ++attrsMap[attr.getName()];
                        if (attrsMap.count(ss.str()) == 0) {
                            newAttr = AttributeDesc(attr.getId(), ss.str(), attr.getType(), attr.getFlags(),
                                attr.getDefaultCompressionMethod(), attr.getAliases(), &attr.getDefaultValue(),
                                attr.getDefaultValueExpr());
                            attrsMap[ss.str()] = 1;
                            break;
                        }
                    }
                }

                outAttrs.push_back(newAttr);
            }

            Dimensions outDims;
            map<string, uint64_t> dimsMap;
            BOOST_FOREACH(const DimensionDesc &dim, srcDesc.getDimensions())
            {
                DimensionDesc newDim;
                if (!dimsMap.count(dim.getBaseName()))
                {
                    dimsMap[dim.getBaseName()] = 1;
                    newDim = DimensionDesc(dim.getBaseName(),
                                           dim.getStartMin(),
                                           dim.getCurrStart(),
                                           dim.getCurrEnd(),
                                           dim.getEndMax(),
                                           dim.getChunkInterval(),
                                           dim.getChunkOverlap());
                }
                else
                {
                    while (true) {
                        stringstream ss;
                        ss << dim.getBaseName() << "_" << ++dimsMap[dim.getBaseName()];
                        if (dimsMap.count(ss.str()) == 0) {
                            newDim = DimensionDesc(ss.str(),
                                                   dim.getStartMin(),
                                                   dim.getCurrStart(),
                                                   dim.getCurrEnd(),
                                                   dim.getEndMax(),
                                                   dim.getChunkInterval(),
                                                   dim.getChunkOverlap());
                            dimsMap[ss.str()] = 1;
                            break;
                        }
                    }
                }

                outDims.push_back(newDim);
            }

         /* Notice that when storing to a non-existant array, we do not propagate the 
            transience of the source array to to the target ...*/

            return ArrayDesc(arrayName, outAttrs, outDims, srcDesc.getFlags() & (~ArrayDesc::TRANSIENT));
        }
示例#14
0
  void DimensionsTest::RunTests()
  {
    Coordinate zero; // [0];
    zero.push_back(0);

    Coordinate one_two; // [1,2]
    one_two.push_back(1);
    one_two.push_back(2);

    Coordinate three_four; // [3,4]
    three_four.push_back(3);
    three_four.push_back(4);
    

    {
      // empty dimensions (unspecified)
      Dimensions d;
      TEST(d.isUnspecified());
      TEST(d.isValid());
      TEST(!d.isDontcare());
      SHOULDFAIL(d.getCount());
      SHOULDFAIL(d.getDimension(0));
      TESTEQUAL("[unspecified]", d.toString());
      SHOULDFAIL(d.getIndex(one_two));
      SHOULDFAIL(d.getCount());
      SHOULDFAIL(d.getDimension(0));
      TESTEQUAL((unsigned int)0, d.getDimensionCount());
    }

    {
      // dontcare dimensions [0]
      Dimensions d;
      d.push_back(0);
      TEST(!d.isUnspecified());
      TEST(d.isDontcare());
      TEST(d.isValid());
      TESTEQUAL("[dontcare]", d.toString());
      SHOULDFAIL(d.getIndex(zero));
      SHOULDFAIL(d.getCount());
      TESTEQUAL((unsigned int)0, d.getDimension(0));
      TESTEQUAL((unsigned int)1, d.getDimensionCount());
    }


    {
      // invalid dimensions
      Dimensions d;
      d.push_back(1);
      d.push_back(0);
      TEST(!d.isUnspecified());
      TEST(!d.isDontcare());
      TEST(!d.isValid());
      TESTEQUAL("[1 0] (invalid)", d.toString());
      SHOULDFAIL(d.getIndex(one_two));
      SHOULDFAIL(d.getCount());
      TESTEQUAL((unsigned int)1, d.getDimension(0));
      TESTEQUAL((unsigned int)0, d.getDimension(1));
      SHOULDFAIL(d.getDimension(2));
      TESTEQUAL((unsigned int)2, d.getDimensionCount());
    }

    {
      // valid dimensions [2,3]
      // two rows, three columns
      Dimensions d;
      d.push_back(2);
      d.push_back(3);
      TEST(!d.isUnspecified());
      TEST(!d.isDontcare());
      TEST(d.isValid());
      TESTEQUAL("[2 3]", d.toString());
      TESTEQUAL((unsigned int)2, d.getDimension(0));
      TESTEQUAL((unsigned int)3, d.getDimension(1));
      SHOULDFAIL(d.getDimension(2));
      TESTEQUAL((unsigned int)6, d.getCount());
      TESTEQUAL((unsigned int)5, d.getIndex(one_two));
      TESTEQUAL((unsigned int)2, d.getDimensionCount());
    }

    {
      //check a two dimensional matrix for proper x-major ordering
      std::vector<size_t> x;
      x.push_back(4);
      x.push_back(5);
      Dimensions d(x);
      size_t testDim1 = 4;
      size_t testDim2 = 5;
      for(size_t i = 0; i < testDim1; i++)
      {
        for(size_t j = 0; j < testDim2; j++)
        {
          Coordinate testCoordinate;
          testCoordinate.push_back(i);
          testCoordinate.push_back(j);

          TESTEQUAL(i+j*testDim1, d.getIndex(testCoordinate));
          TESTEQUAL(vecToString(testCoordinate),
                    vecToString(d.getCoordinate(i+j*testDim1)));
        }
      }
    }

    {
      //check a three dimensional matrix for proper x-major ordering
      std::vector<size_t> x;
      x.push_back(3);
      x.push_back(4);
      x.push_back(5);
      Dimensions d(x);
      size_t testDim1 = 3;
      size_t testDim2 = 4;
      size_t testDim3 = 5;
      for(size_t i = 0; i < testDim1; i++)
      {
        for(size_t j = 0; j < testDim2; j++)
        {
          for(size_t k = 0; k < testDim3; k++)
          {
            Coordinate testCoordinate;
            testCoordinate.push_back(i);
            testCoordinate.push_back(j);
            testCoordinate.push_back(k);

            TESTEQUAL(i +
                      j*testDim1 +
                      k*testDim1*testDim2, d.getIndex(testCoordinate));

            TESTEQUAL(vecToString(testCoordinate),
                      vecToString(d.getCoordinate(i +
                                                  j*testDim1 +
                                                  k*testDim1*testDim2)));
          }
        }
      }
    }

    { 
      // alternate constructor
      std::vector<size_t> x;
      x.push_back(2);
      x.push_back(5);
      Dimensions d(x);
      TEST(!d.isUnspecified());
      TEST(!d.isDontcare());
      TEST(d.isValid());
      
      TESTEQUAL((unsigned int)2, d.getDimension(0));
      TESTEQUAL((unsigned int)5, d.getDimension(1));
      SHOULDFAIL(d.getDimension(2));
      TESTEQUAL((unsigned int)2, d.getDimensionCount());
    }

  } // RunTests()
示例#15
0
TEST(WatcherTest, SampleNetwork)
{
  //generate sample network
  Network n;
  n.addRegion("level1", "TestNode", "");
  n.addRegion("level2", "TestNode", "");
  n.addRegion("level3", "TestNode", "");
  Dimensions d;
  d.push_back(8);
  d.push_back(4);
  n.getRegions().getByName("level1")->setDimensions(d);
  n.link("level1", "level2", "TestFanIn2", "");
  n.link("level2", "level3", "TestFanIn2", "");
  n.initialize();

  //erase any previous contents of testfile
  OFStream o("testfile");
  o.close();
  
  //test creation
  Watcher w("testfile");

  //test uint32Params
  unsigned int id1 = w.watchParam("level1", "uint32Param");
  ASSERT_EQ(id1, (unsigned int)1);
  //test uint64Params
  unsigned int id2 = w.watchParam("level1", "uint64Param");
  ASSERT_EQ(id2, (unsigned int)2);
  //test int32Params
  w.watchParam("level1", "int32Param");
  //test int64Params
  w.watchParam("level1", "int64Param");
  //test real32Params
  w.watchParam("level1", "real32Param");
  //test real64Params
  w.watchParam("level1", "real64Param");
  //test stringParams
  w.watchParam("level1", "stringParam");
  //test unclonedParams
  w.watchParam("level1", "unclonedParam", 0);
  w.watchParam("level1", "unclonedParam", 1);
  
  //test attachToNetwork()
  w.attachToNetwork(n);

  //test two simultaneous Watchers on the same network with different files
  Watcher* w2 = new Watcher("testfile2");

  //test int64ArrayParam
  w2->watchParam("level1", "int64ArrayParam");
  //test real32ArrayParam
  w2->watchParam("level1", "real32ArrayParam");
  //test output
  w2->watchOutput("level1", "bottomUpOut");
  //test int64ArrayParam, sparse = false
  w2->watchParam("level1", "int64ArrayParam", -1, false);

  w2->attachToNetwork(n);

  //set one of the uncloned parameters to 1 instead of 0
  //n.getRegions().getByName("level1")->getNodeAtIndex(1).setParameterUInt32("unclonedParam", (UInt32)1);
  //n.run(3);
  //see if Watcher notices change in parameter values after 3 iterations
  n.getRegions().getByName("level1")->setParameterUInt64("uint64Param", (UInt64)66);
  n.run(3);

  //test flushFile() - this should produce output
  w.flushFile();

  //test closeFile()
  w.closeFile();

  //test to make sure data is flushed when Watcher is deleted
  delete w2;
}