VectorFileEffector::VectorFileEffector(const ValueMap& params, Region* region) :
RegionImpl(region),
dataIn_(NTA_BasicType_Real32),
filename_(""),
outFile_(0)
{
if (params.contains("outputFile"))
filename_ = *params.getString("outputFile");
else
filename_ = "";
}
TEST(ValueTest, ValueMap)
{
boost::shared_ptr<Scalar> s(new Scalar(NTA_BasicType_Int32));
s->value.int32 = 10;
boost::shared_ptr<Array> a(new Array(NTA_BasicType_Real32));
boost::shared_ptr<std::string> str(new std::string("hello world"));
ValueMap vm;
vm.add("scalar", s);
vm.add("array", a);
vm.add("string", str);
ASSERT_ANY_THROW(vm.add("scalar", s));
ASSERT_TRUE(vm.contains("scalar"));
ASSERT_TRUE(vm.contains("array"));
ASSERT_TRUE(vm.contains("string"));
ASSERT_TRUE(!vm.contains("foo"));
ASSERT_TRUE(!vm.contains("scalar2"));
ASSERT_TRUE(!vm.contains("xscalar"));
boost::shared_ptr<Scalar> s1 = vm.getScalar("scalar");
ASSERT_TRUE(s1 == s);
boost::shared_ptr<Array> a1 = vm.getArray("array");
ASSERT_TRUE(a1 == a);
boost::shared_ptr<Scalar> def(new Scalar(NTA_BasicType_Int32));
Int32 x = vm.getScalarT("scalar", (Int32)20);
ASSERT_EQ((Int32)10, x);
x = vm.getScalarT("scalar2", (Int32)20);
ASSERT_EQ((Int32)20, x);
Value v = vm.getValue("array");
ASSERT_EQ(Value::arrayCategory, v.getCategory());
ASSERT_TRUE(v.getArray() == a);
}
/*
* For converting param specs for Regions and LinkPolicies
*/
ValueMap toValueMap(const char* yamlstring,
Collection<ParameterSpec>& parameters,
const std::string & nodeType,
const std::string & regionName)
{
ValueMap vm;
// yaml-cpp bug: append a space if it is only one character
// This is very inefficient, but should be ok since it is
// just used at construction time for short strings
std::string paddedstring(yamlstring);
// TODO: strip white space to determine if empty
bool empty = (paddedstring.size() == 0);
if (paddedstring.size() < 2)
paddedstring = paddedstring + " ";
std::stringstream s(paddedstring);
// IMemStream s(yamlstring, ::strlen(yamlstring));
// TODO: utf-8 compatible?
YAML::Node doc;
if (!empty)
{
YAML::Parser parser(s);
bool success = parser.GetNextDocument(doc);
if (!success)
NTA_THROW << "Unable to find document in YAML string";
// A ValueMap is specified as a dictionary
if (doc.Type() != YAML::NodeType::Map)
{
std::string ys(yamlstring);
if (ys.size() > 30)
{
ys = ys.substr(0, 30) + "...";
}
NTA_THROW << "YAML string '" << ys
<< "' does not not specify a dictionary of key-value pairs. "
<< "Region and Link parameters must be specified at a dictionary";
}
}
// Grab each value out of the YAML dictionary and put into the ValueMap
// if it is allowed by the nodespec.
YAML::Iterator i;
for (i = doc.begin(); i != doc.end(); i++)
{
const std::string key = i.first().to<std::string>();
if (!parameters.contains(key))
{
std::stringstream ss;
for (UInt j = 0; j < parameters.getCount(); j++)
{
ss << " " << parameters.getByIndex(j).first << "\n";
}
if (nodeType == std::string(""))
{
NTA_THROW << "Unknown parameter '" << key << "'\n"
<< "Valid parameters are:\n" << ss.str();
}
else
{
NTA_CHECK(regionName != std::string(""));
NTA_THROW << "Unknown parameter '" << key << "' for region '"
<< regionName << "' of type '" << nodeType << "'\n"
<< "Valid parameters are:\n" << ss.str();
}
}
if (vm.contains(key))
NTA_THROW << "Parameter '" << key << "' specified more than once in YAML document";
ParameterSpec spec = parameters.getByName(key);
try
{
Value v = toValue(i.second(), spec.dataType);
if (v.isScalar() && spec.count != 1)
{
throw std::runtime_error("Expected array value but got scalar value");
}
if (!v.isScalar() && spec.count == 1)
{
throw std::runtime_error("Expected scalar value but got array value");
}
vm.add(key, v);
} catch (std::runtime_error& e) {
NTA_THROW << "Unable to set parameter '" << key << "'. " << e.what();
}
}
// Populate ValueMap with default values if they were not specified in the YAML dictionary.
for (size_t i = 0; i < parameters.getCount(); i++)
{
std::pair<std::string, ParameterSpec>& item = parameters.getByIndex(i);
if (!vm.contains(item.first))
{
ParameterSpec & ps = item.second;
if (ps.defaultValue != "")
{
// TODO: This check should be uncommented after dropping NuPIC 1.x nodes (which don't comply)
// if (ps.accessMode != ParameterSpec::CreateAccess)
// {
// NTA_THROW << "Default value for non-create parameter: " << item.first;
// }
try {
#ifdef YAMLDEBUG
NTA_DEBUG << "Adding default value '" << ps.defaultValue
<< "' to parameter " << item.first
<< " of type " << BasicType::getName(ps.dataType)
<< " count " << ps.count;
#endif
Value v = toValue(ps.defaultValue, ps.dataType);
vm.add(item.first, v);
} catch (...) {
NTA_THROW << "Unable to set default value for item '"
<< item.first << "' of datatype "
<< BasicType::getName(ps.dataType)
<<" with value '" << ps.defaultValue << "'";
}
}
}
}
return vm;
}
TEST(YAMLUtilsTest, ParameterSpec)
{
Collection<ParameterSpec> ps;
ps.add(
"int32Param",
ParameterSpec(
"Int32 scalar parameter", // description
NTA_BasicType_Int32,
1, // elementCount
"", // constraints
"32", // defaultValue
ParameterSpec::ReadWriteAccess));
ps.add(
"uint32Param",
ParameterSpec(
"UInt32 scalar parameter", // description
NTA_BasicType_UInt32,
1, // elementCount
"", // constraints
"33", // defaultValue
ParameterSpec::ReadWriteAccess));
ps.add(
"int64Param",
ParameterSpec(
"Int64 scalar parameter", // description
NTA_BasicType_Int64,
1, // elementCount
"", // constraints
"64", // defaultValue
ParameterSpec::ReadWriteAccess));
ps.add(
"uint64Param",
ParameterSpec(
"UInt64 scalar parameter", // description
NTA_BasicType_UInt64,
1, // elementCount
"", // constraints
"65", // defaultValue
ParameterSpec::ReadWriteAccess));
ps.add(
"real32Param",
ParameterSpec(
"Real32 scalar parameter", // description
NTA_BasicType_Real32,
1, // elementCount
"", // constraints
"32.1", // defaultValue
ParameterSpec::ReadWriteAccess));
ps.add(
"real64Param",
ParameterSpec(
"Real64 scalar parameter", // description
NTA_BasicType_Real64,
1, // elementCount
"", // constraints
"64.1", // defaultValue
ParameterSpec::ReadWriteAccess));
ps.add(
"real32ArrayParam",
ParameterSpec(
"int32 array parameter",
NTA_BasicType_Real32,
0, // array
"",
"",
ParameterSpec::ReadWriteAccess));
ps.add(
"int64ArrayParam",
ParameterSpec(
"int64 array parameter",
NTA_BasicType_Int64,
0, // array
"",
"",
ParameterSpec::ReadWriteAccess));
ps.add(
"computeCallback",
ParameterSpec(
"address of a function that is called at every compute()",
NTA_BasicType_Handle,
1,
"",
"", // handles must not have a default value
ParameterSpec::ReadWriteAccess));
ps.add(
"stringParam",
ParameterSpec(
"string parameter",
NTA_BasicType_Byte,
0, // length=0 required for strings
"",
"default value",
ParameterSpec::ReadWriteAccess));
ps.add(
"boolParam",
ParameterSpec(
"bool parameter",
NTA_BasicType_Bool,
1,
"",
"false",
ParameterSpec::ReadWriteAccess));
NTA_DEBUG << "ps count: " << ps.getCount();
ValueMap vm = YAMLUtils::toValueMap("", ps);
EXPECT_TRUE(vm.contains("int32Param"))
<< "assertion vm.contains(\"int32Param\") failed at "
<< __FILE__ << ":" << __LINE__ ;
ASSERT_EQ((Int32)32, vm.getScalarT<Int32>("int32Param"));
EXPECT_TRUE(vm.contains("boolParam"))
<< "assertion vm.contains(\"boolParam\") failed at "
<< __FILE__ << ":" << __LINE__ ;
ASSERT_EQ(false, vm.getScalarT<bool>("boolParam"));
// disabled until we fix default string params
// TEST(vm.contains("stringParam"));
// EXPECT_STREQ("default value", vm.getString("stringParam")->c_str());
// Test error message in case of invalid parameter with and without nodeType and regionName
try
{
YAMLUtils::toValueMap("{ blah: True }", ps, "nodeType", "regionName");
}
catch (nupic::Exception & e)
{
std::string s("Unknown parameter 'blah' for region 'regionName'");
EXPECT_TRUE(std::string(e.getMessage()).find(s) == 0)
<< "assertion std::string(e.getMessage()).find(s) == 0 failed at "
<< __FILE__ << ":" << __LINE__ ;
}
try
{
YAMLUtils::toValueMap("{ blah: True }", ps);
}
catch (nupic::Exception & e)
{
std::string s("Unknown parameter 'blah'\nValid");
EXPECT_TRUE(std::string(e.getMessage()).find(s) == 0)
<< "assertion std::string(e.getMessage()).find(s) == 0 failed at "
<< __FILE__ << ":" << __LINE__ ;
}
}
/*
* For converting param specs for Regions and LinkPolicies
*/
ValueMap toValueMap(const char *yamlstring,
Collection<ParameterSpec> ¶meters,
const std::string &nodeType,
const std::string ®ionName) {
ValueMap vm;
// special value that applies to all regions.
ParameterSpec dim_spec("Buffer dimensions for region's global dimensions. "
"Syntax: {dim: [2,3]}", // description
NTA_BasicType_UInt32,
0, // elementCount (an array of unknown size)
"", // constraints
"", // defaultValue
ParameterSpec::ReadWriteAccess);
std::string paddedstring(yamlstring);
// TODO: strip white space to determine if empty
bool empty = (paddedstring.size() == 0);
// TODO: utf-8 compatible?
const YAML::Node doc = YAML::Load(paddedstring);
if(!empty) {
// A ValueMap is specified as a dictionary
if (doc.Type() != YAML::NodeType::Map) {
std::string ys(yamlstring);
if (ys.size() > 30) {
ys = ys.substr(0, 30) + "...";
}
NTA_THROW
<< "YAML string '" << ys
<< "' does not not specify a dictionary of key-value pairs. "
<< "Region and Link parameters must be specified as a dictionary";
}
}
// Grab each value out of the YAML dictionary and put into the ValueMap
// if it is allowed by the nodespec.
for (auto i = doc.begin(); i != doc.end(); i++)
{
ParameterSpec ps;
const auto key = i->first.as<std::string>();
if (key == "dim")
ps = dim_spec;
else {
if (!parameters.contains(key))
{
std::stringstream ss;
for (UInt j = 0; j < parameters.getCount(); j++){
ss << " " << parameters.getByIndex(j).first << "\n";
}
if (nodeType == std::string("")) {
NTA_THROW << "Unknown parameter '" << key << "'\n"
<< "Valid parameters are:\n" << ss.str();
} else {
NTA_CHECK(regionName != std::string(""));
NTA_THROW << "Unknown parameter '" << key << "' for region '"
<< regionName << "' of type '" << nodeType << "'\n"
<< "Valid parameters are:\n"
<< ss.str();
}
}
ps = parameters.getByName(key); // makes a copy of ParameterSpec
}
if (vm.contains(key))
NTA_THROW << "Parameter '" << key << "' specified more than once in YAML document";
try
{
if (ps.accessMode == ParameterSpec::ReadOnlyAccess) {
NTA_THROW << "Parameter '" << key << "'. This is ReadOnly access. Cannot be set.";
}
Value v = toValue(i->second, ps.dataType);
if (v.isScalar() && ps.count != 1)
{
NTA_THROW << "Parameter '" << key << "'. Bad value in runtime parameters. Expected array value but got scalar value";
}
if (!v.isScalar() && ps.count == 1)
{
NTA_THROW << "Parameter '" << key << "'. Bad value in runtime parameters. Expected scalar value but got array value";
}
vm.add(key, v);
} catch (std::runtime_error &e) {
NTA_THROW << "Unable to set parameter '" << key << "'. " << e.what();
}
} //end for
// Populate ValueMap with default values if they were not specified in the YAML dictionary.
for (size_t i = 0; i < parameters.getCount(); i++)
{
const std::pair<std::string, ParameterSpec>& item = parameters.getByIndex(i);
if (!vm.contains(item.first))
{
const ParameterSpec & ps = item.second;
if (ps.defaultValue != "")
{
// TODO: This check should be uncommented after dropping NuPIC 1.x nodes (which don't comply) //FIXME try this
// if (ps.accessMode != ParameterSpec::CreateAccess)
// {
// NTA_THROW << "Default value for non-create parameter: " << item.first;
// }
try {
#ifdef YAMLDEBUG
NTA_DEBUG << "Adding default value '" << ps.defaultValue
<< "' to parameter " << item.first << " of type "
<< BasicType::getName(ps.dataType) << " count " << ps.count;
#endif
// NOTE: this can handle both scalers and arrays
// Arrays MUST be in Yaml sequence format even if one element.
// i.e. [1,2,3]
Value v = toValue(ps.defaultValue, ps.dataType);
if (v.isScalar() && ps.count != 1)
{
NTA_THROW << "Parameter '" << item.first << "'. Bad default value in spec. Expected array value but got scalar value";
}
if (!v.isScalar() && ps.count == 1)
{
NTA_THROW << "Parameter '" << item.first << "'. Bad default value in spec. Expected scalar value but got array value";
}
vm.add(item.first, v);
} catch (...) {
NTA_THROW << "Unable to set default value for item '" << item.first
<< "' of datatype " << BasicType::getName(ps.dataType)
<< " with value '" << ps.defaultValue << "'";
}
}
}
}
return vm;
}
void ValueTest::RunTests()
{
// scalar
{
boost::shared_ptr<Scalar> s(new Scalar(NTA_BasicType_Int32));
s->value.int32 = 10;
Value v(s);
TEST(v.isScalar());
TEST(! v.isString());
TEST(! v.isArray());
TESTEQUAL(Value::scalarCategory, v.getCategory());
TESTEQUAL(NTA_BasicType_Int32, v.getType());
boost::shared_ptr<Scalar> s1 = v.getScalar();
TEST(s1 == s);
SHOULDFAIL(v.getArray());
SHOULDFAIL(v.getString());
TESTEQUAL("Scalar of type Int32", v.getDescription());
Int32 x = v.getScalarT<Int32>();
TESTEQUAL(10, x);
SHOULDFAIL(v.getScalarT<UInt32>());
}
// array
{
boost::shared_ptr<Array> s(new Array(NTA_BasicType_Int32));
s->allocateBuffer(10);
Value v(s);
TEST(v.isArray());
TEST(! v.isString());
TEST(! v.isScalar());
TESTEQUAL(Value::arrayCategory, v.getCategory());
TESTEQUAL(NTA_BasicType_Int32, v.getType());
boost::shared_ptr<Array> s1 = v.getArray();
TEST(s1 == s);
SHOULDFAIL(v.getScalar());
SHOULDFAIL(v.getString());
SHOULDFAIL(v.getScalarT<Int32>());
TESTEQUAL("Array of type Int32", v.getDescription());
}
// string
{
boost::shared_ptr<std::string> s(new std::string("hello world"));
Value v(s);
TEST(! v.isArray());
TEST(v.isString());
TEST(! v.isScalar());
TESTEQUAL(Value::stringCategory, v.getCategory());
TESTEQUAL(NTA_BasicType_Byte, v.getType());
boost::shared_ptr<std::string> s1 = v.getString();
TESTEQUAL("hello world", s1->c_str());
SHOULDFAIL(v.getScalar());
SHOULDFAIL(v.getArray());
SHOULDFAIL(v.getScalarT<Int32>());
TESTEQUAL("string (hello world)", v.getDescription());
}
// ValueMap
{
boost::shared_ptr<Scalar> s(new Scalar(NTA_BasicType_Int32));
s->value.int32 = 10;
boost::shared_ptr<Array> a(new Array(NTA_BasicType_Real32));
boost::shared_ptr<std::string> str(new std::string("hello world"));
ValueMap vm;
vm.add("scalar", s);
vm.add("array", a);
vm.add("string", str);
SHOULDFAIL(vm.add("scalar", s));
TEST(vm.contains("scalar"));
TEST(vm.contains("array"));
TEST(vm.contains("string"));
TEST(!vm.contains("foo"));
TEST(!vm.contains("scalar2"));
TEST(!vm.contains("xscalar"));
boost::shared_ptr<Scalar> s1 = vm.getScalar("scalar");
TEST(s1 == s);
boost::shared_ptr<Array> a1 = vm.getArray("array");
TEST(a1 == a);
boost::shared_ptr<Scalar> def(new Scalar(NTA_BasicType_Int32));
Int32 x = vm.getScalarT("scalar", (Int32)20);
TESTEQUAL((Int32)10, x);
x = vm.getScalarT("scalar2", (Int32)20);
TESTEQUAL((Int32)20, x);
Value v = vm.getValue("array");
TESTEQUAL(Value::arrayCategory, v.getCategory());
TEST(v.getArray() == a);
}
}
/*
* this method saves the attribute together with the host string that
* defines the type of object that this attribute is associated to (like
* position or document) and the hosts database id.
*/
void AttributeMap::save( dbID id )
{
checkHost();
QSqlQuery attribQuery;
attribQuery.prepare( "SELECT id, valueIsList FROM attributes WHERE hostObject=:host AND hostId=:hostId AND name=:name" );
attribQuery.bindValue( ":host", mHost );
attribQuery.bindValue( ":hostId", id.toString() );
Iterator it;
for ( it = begin(); it != end(); ++it ) {
Attribute att = it.value();
kDebug() << ">> oo- saving attribute with name " << it.key() << " for " << id.toString() << " att-name: " << att.name();
attribQuery.bindValue( ":name", att.name() );
attribQuery.exec();
QString attribId;
if ( attribQuery.next() ) {
// the attrib exists. Check the values
attribId = attribQuery.value(0).toString(); // the id
if ( att.value().isNull() || att.mDelete ) {
// the value is empty. the existing entry needs to be dropped
dbDeleteAttribute( attribId );
return;
}
} else {
// the attrib does not yet exist. Create if att value is not null.
if ( att.value().isNull() ) {
kDebug() << "oo- skip writing of attribute, value is empty";
} else {
kDebug() << "oo- writing of attribute name " << att.name();
QSqlQuery insQuery;
insQuery.prepare( "INSERT INTO attributes (hostObject, hostId, name, valueIsList, relationTable, "
"relationIDColumn, relationStringColumn) "
"VALUES (:host, :hostId, :name, :valueIsList, :relTable, :relIDCol, :relStringCol )" );
insQuery.bindValue( ":host", mHost );
insQuery.bindValue( ":hostId", id.toString() );
insQuery.bindValue( ":name", att.name() );
insQuery.bindValue( ":valueIsList", att.listValue() );
// Write the relation table info. These remain empty for non related attributes.
insQuery.bindValue( ":relTable", att.mTable );
insQuery.bindValue( ":relIDCol", att.mIdCol );
insQuery.bindValue( ":relStringCol", att.mStringCol );
insQuery.exec();
dbID attId = KraftDB::self()->getLastInsertID();
attribId = attId.toString();
}
}
// store the id to be able to drop not longer existent values
kDebug() << "adding attribute id " << attribId << " for attribute " << att.name();
// now there is a valid entry in the attribute table. Check the values.
QSqlQuery valueQuery( "SELECT id, value FROM attributeValues WHERE attributeId=" + attribId );
typedef QMap<QString, QString> ValueMap;
ValueMap valueMap;
while ( valueQuery.next() ) {
QString idStr = valueQuery.value( 0 ).toString(); // id
QString valStr = valueQuery.value( 1 ).toString(); // value
valueMap[valStr] = idStr;
}
// create a stringlist with the current values of the attribute
if ( att.listValue() ) {
QStringList newValues;
newValues = att.mValue.toStringList();
kDebug() << "new values are: " << newValues.join( ", " );
if ( newValues.empty() ) {
// delete the entire attribute.
dbDeleteValue( attribId ); // deletes all values
dbDeleteAttribute( attribId );
valueMap.clear();
} else {
// we really have new values
QSqlQuery insValue;
insValue.prepare( "INSERT INTO attributeValues (attributeId, value) VALUES (:attribId, :val)" );
insValue.bindValue( ":attribId", attribId );
// loop over all existing new values of the attribute.
for ( QStringList::Iterator valIt = newValues.begin(); valIt != newValues.end(); ++valIt ) {
QString curValue = *valIt;
if ( valueMap.contains( curValue ) ) {
// the valueMap is already saved. remove it from the valueMap string
kDebug() << "Value " << curValue << " is already present with id " << valueMap[curValue];
valueMap.remove( curValue );
} else {
// the value is not yet there, insert it.
insValue.bindValue( ":val", curValue );
insValue.exec();
}
}
}
} else {
// only a single entry for the attribte, update if needed.
QString newValue = att.mValue.toString(); // access the attribute object directly to get the numeric
kDebug() << "NEW value String: " << newValue;
// value in case the attribute is bound to a relation table
if ( newValue.isEmpty() ) {
// delete the entire attribute
dbDeleteValue( attribId ); // deletes all values
dbDeleteAttribute( attribId );
valueMap.clear();
} else {
if ( valueMap.empty() ) {
// there is no entry yet that could be updated.
QSqlQuery insertQuery;
insertQuery.prepare( "INSERT INTO attributeValues (attributeId, value ) VALUES (:id, :val)" );
insertQuery.bindValue( ":id", attribId );
insertQuery.bindValue( ":val", newValue );
insertQuery.exec();
kDebug() << "insert new attrib value for non list: " << newValue;
} else {
QString oldValue = valueMap.begin().key();
QString id = valueMap.begin().value();
if ( newValue != oldValue ) {
kDebug() << "Updating " << id << " from " << oldValue << " to " << newValue;
QSqlQuery updateQuery;
updateQuery.prepare( "UPDATE attributeValues SET value=:val WHERE id=:id" );
updateQuery.bindValue( ":val", newValue );
updateQuery.bindValue( ":id", id );
kDebug() << "do the update!";
updateQuery.exec();
}
valueMap.remove( oldValue );
}
}
}
// remove all still existing entries in the valueMap because they point to values which are
// in the db but were deleted from the attribute
if ( ! valueMap.isEmpty() ) {
ValueMap::Iterator mapIt;
for ( mapIt = valueMap.begin(); mapIt != valueMap.end(); ++mapIt ) {
QString valId = mapIt.value();
dbDeleteValue( attribId, valId );
}
}
}
}
