ParaFileData::ParaFileData (const libconfig::Config &cfg, const string token)
{ string tmp0 = cfg.lookup (token+".type"); type = tmp0;
string tmp1 = cfg.lookup (token+".data"); data = tmp1;
string tmp2 = cfg.lookup (token+".label"); label = tmp2;
string tmp3 = cfg.lookup ("statdata.mean" ); mean = tmp3;
LOG (INFO) << "\tdata path\t" << data;
};
TensorFormat::TensorFormat (const libconfig::Config &cfg) : isTrain(true)
{ rows = cfg.lookup ("tformat.rows");
cols = cfg.lookup ("tformat.cols");
chls = cfg.lookup ("tformat.chls");
nums = cfg.lookup ("tformat.nums");
numBatch = cfg.lookup ("tformat.numBatch");
numClass = cfg.lookup ("tformat.numClass");
};
ParaDBData::ParaDBData (const libconfig::Config &cfg, const string token)
{ string tmp1 = cfg.lookup (token+".path"); path = tmp1;
numRows = cfg.lookup (token+".numRows");
threads = cfg.lookup (token+".threads");
verbose = cfg.lookup (token+".verbose");
};
void ParaModel::set_para (const libconfig::Config &cfg)
{ if_train = cfg.lookup ("model.if_train");
if_test = cfg.lookup ("model.if_test");
if_update = cfg.lookup ("model.if_update");
string tmp1 = cfg.lookup ("model.path"); path = tmp1;
};
void constructsubtree(
libconfig::Config& cfg,
libconfig::Setting& cfgnode,
Operator** rootaddr, //< output
Query::UserDefinedOpMapT& udops,
int level,
Query::OperatorDepthT& depthmap //< output
)
{
// Lookup name and type.
//
string name = cfgnode["name"];
string type = cfg.getRoot()[name]["type"];
assert(*rootaddr == 0);
// Based on type:
// 1. Allocate and chain in appropriate object
// 2. Check syntax for child
// 3. sanitycheck() branch
// 4. Recursively call self for children
// 5. Initialize this node
//
if ( type == "scan"
|| type == "partitionedscan"
|| type == "parallelscan"
|| type == "generator_int"
|| type == "generator_long"
#ifdef ENABLE_HDF5
|| type == "hdf5scan"
#ifdef ENABLE_FASTBIT
|| type == "hdf5index"
|| type == "hdf5random"
#endif
#endif
#ifdef ENABLE_FASTBIT
|| type == "fastbitscan"
#endif
)
{
// Scan operator, no children.
//
ZeroInputOp* tmp = NULL;
if (type == "scan")
tmp = new ScanOp();
else if (type == "partitionedscan")
tmp = new PartitionedScanOp();
else if (type == "parallelscan")
tmp = new ParallelScanOp();
else if (type == "generator_int")
tmp = new IntGeneratorOp();
else if (type == "generator_long")
tmp = new LongGeneratorOp();
#ifdef ENABLE_HDF5
else if (type == "hdf5scan")
tmp = new ScanHdf5Op();
#ifdef ENABLE_FASTBIT
else if (type == "hdf5index")
tmp = new IndexHdf5Op();
else if (type == "hdf5random")
tmp = new RandomLookupsHdf5Op();
#endif
#endif
#ifdef ENABLE_FASTBIT
else if (type == "fastbitscan")
tmp = new FastBitScanOp();
#endif
(*rootaddr) = tmp;
depthmap[tmp] = level;
}
else if ( type == "hashjoin"
|| type == "sortmergejoin"
|| type == "mpsmjoin"
|| type == "newmpsmjoin"
|| type == "preprejoin"
|| type == "indexhashjoin"
)
{
// Dual-input operator
//
DualInputOp* tmp = NULL;
if (type == "hashjoin")
tmp = new HashJoinOp();
else if (type == "sortmergejoin")
tmp = new SortMergeJoinOp();
else if (type == "mpsmjoin")
tmp = new OldMPSMJoinOp();
else if (type == "newmpsmjoin")
tmp = new MPSMJoinOp();
else if (type == "preprejoin")
tmp = new PresortedPrepartitionedMergeJoinOp();
else if (type == "indexhashjoin")
tmp = new IndexHashJoinOp();
(*rootaddr) = tmp;
depthmap[tmp] = level;
sanitycheck(cfg, cfgnode, "build");
constructsubtree(cfg, cfgnode["build"], &(tmp->buildOp), udops, level+1, depthmap);
sanitycheck(cfg, cfgnode, "probe");
constructsubtree(cfg, cfgnode["probe"], &(tmp->probeOp), udops, level+1, depthmap);
}
else
{
// Single-input operator
//
SingleInputOp* tmp = NULL;
if (type == "aggregate_sum")
tmp = new AggregateSum();
else if (type == "aggregate_count")
tmp = new AggregateCount();
else if (type == "merge")
tmp = new MergeOp();
else if (type == "shmwriter")
tmp = new MemSegmentWriter();
else if (type == "filter")
tmp = new Filter();
else if (type == "cycle_accountant")
tmp = new CycleAccountant();
else if (type == "projection")
tmp = new Project();
else if (type == "checker_callstate")
tmp = new CallStateChecker();
else if (type == "printer_schema")
tmp = new SchemaPrinter();
else if (type == "printer_tuplecount")
tmp = new TupleCountPrinter();
else if (type == "printer_perfcount")
tmp = new PerfCountPrinter();
else if (type == "sort")
tmp = new SortLimit();
else if (type == "printer_bitentropy")
tmp = new BitEntropyPrinter();
else if (type == "consumer")
tmp = new ConsumeOp();
else if (type == "printer_callcount")
tmp = new CallCountPrinter();
else if (type == "threadidprepend")
tmp = new ThreadIdPrependOp();
else if (type == "partition")
tmp = new PartitionOp();
else
{
// It's a user-defined type?
//
Query::UserDefinedOpMapT::iterator it;
it = udops.find(type);
// Not a user-defined type, no idea what is it.
//
if (it == udops.end())
throw MissingParameterException("`" + type + "' is neither a built-in nor a user-defined type.");
// Claim ownership of operator.
//
tmp = it->second;
it->second = 0;
}
(*rootaddr) = tmp;
depthmap[tmp] = level;
sanitycheck(cfg, cfgnode, "input");
constructsubtree(cfg, cfgnode["input"], &(tmp->nextOp), udops, level+1, depthmap);
}
// Call Operator::init on this node.
//
assert(*rootaddr != 0);
(*rootaddr)->init(cfg, cfg.lookup(name));
}
/**
* Constructs a query tree from the specified configuration file.
* The pre-allocated user-defined operators are used if an operator type is
* unknown. If a pre-allocated operator is used, its entry in the OpMap is
* emptied, and it's now the responsibility of the Query to destroy the object.
*
* @param cfg Configuration file to initialize tree with.
* @param udops User-defined operator map. Each operator must have been
* allocated with new. If it is used, the entry is set to NULL, and then
* it becomes the Query responsibility to call delete on the operator
* object.
*/
void Query::create(libconfig::Config& cfg, UserDefinedOpMapT& udops)
{
sanitycheck(cfg, cfg.getRoot(), "treeroot");
constructsubtree(cfg, cfg.lookup("treeroot"), &tree, udops, 0, operatorDepth);
}