void CoreSchema::initCoreSchema(Schema &schema)
{
assert (schema.size() == 0);
schema.addPrimitiveType<char>("char"); // 0
schema.addPrimitiveType<int8>("int8"); // 1
schema.addPrimitiveType<int16>("int16"); // 2
schema.addPrimitiveType<int32>("int32"); // 3
schema.addPrimitiveType<int64>("int64"); // 4
schema.addPrimitiveType<uint8>("uint8"); // 5
schema.addPrimitiveType<uint16>("uint16"); // 6
schema.addPrimitiveType<uint32>("uint32"); // 7
schema.addPrimitiveType<uint64>("uint64"); // 8
schema.addPrimitiveType<float32>("float32"); // 9
schema.addPrimitiveType<float64>("float64"); // 10
schema.addPrimitiveType<RawPointer>("pointer"); // 11
schema.addPrimitiveType<RawPointer>("tagged_pointer"); // 12
schema << FieldType(); // 13
schema << Field(); // 14
schema << PrimitiveType(); // 15
schema << CustomType(); // 16
schema << Schema(); // 17
// FIRST_USER_TYPEID is 256
schema.setFreeTypeID(FIRST_USER_TYPEID); // from 0x000d to 0x00ff, the types are reserved
// custom types should start from 0x0100
}
void ByteSwap::initialize()
{
Filter::initialize();
const Stage& stage = getPrevStage();
this->setNumPoints(stage.getNumPoints());
this->setPointCountType(stage.getPointCountType());
schema::index_by_index const& dimensions =
m_schema.getDimensions().get<schema::index>();
std::vector<Dimension> new_dimensions;
for (schema::index_by_index::const_iterator i = dimensions.begin(); i != dimensions.end(); ++i)
{
pdal::Dimension d(*i);
pdal::EndianType t = i->getEndianness();
if (t == Endian_Little)
{
d.setEndianness(Endian_Big);
}
else if (t == Endian_Big)
{
d.setEndianness(Endian_Little);
}
else
{
throw pdal_error("ByteSwapFilter can only swap big/little endian dimensions");
}
new_dimensions.push_back(d);
}
m_schema = Schema(new_dimensions);
}
Reader::Reader(const Options& options)
: pdal::Reader(options)
, m_bounds(options.getValueOrThrow<Bounds<double> >("bounds"))
, m_numPoints(options.getValueOrThrow<boost::uint64_t>("num_points"))
, m_mode(string2mode(options.getValueOrThrow<std::string>("mode")))
{
m_schema = Schema(getDefaultDimensions());
}
Reader::Reader(const Bounds<double>& bounds, boost::uint64_t numPoints, Mode mode, const std::vector<Dimension>& dimensions)
: pdal::Reader(Options::none())
, m_bounds(bounds)
, m_numPoints(numPoints)
, m_mode(mode)
{
m_schema = Schema(dimensions);
}
Reader::Reader(const Bounds<double>& bounds, boost::uint64_t numPoints, Mode mode)
: pdal::Reader(Options::none())
, m_bounds(bounds)
, m_numPoints(numPoints)
, m_mode(mode)
{
m_schema = Schema(getDefaultDimensions());
}
int main(int argc, char* argv[])
{
TEnv Env(argc, argv);
TStr PrefixPath = Env.GetArgs() > 1 ? Env.GetArg(1) : TStr("");
double ts1 = Tick();
TTableContext Context;
TVec<TPair<PTable,TStr> > NodeTblV = TVec<TPair<PTable,TStr> >();
TVec<TPair<PTable, int> > EdgeTblV = TVec<TPair<PTable, int> >();
Schema NodeSchema = Schema();
Schema EdgeSchema = Schema();
LoadFlickrTables(PrefixPath, Context, NodeTblV, NodeSchema, EdgeTblV, EdgeSchema);
double ts2 = Tick();
THash<TStr,TStrH> NStrH;
TIntStrH NIdH;
CreateIdHashes(NodeTblV, NStrH, NIdH);
double ts3 = Tick();
PSVNet Graph = LoadGraphMNet<PSVNet>(NodeTblV, EdgeTblV, NStrH, NIdH);
double ts4 = Tick();
int nExps = 10;
int nTriads = 0;
for (int i = 0; i < nExps; i++) {
nTriads = TSnap::GetTriads(Graph);
}
double ts5 = Tick();
StdOut->PutStrFmtLn("Triads %d", nTriads);
PSOut TimeOut = TFOut::New(PrefixPath + TStr("time.txt"), true);
TimeOut->PutStrFmtLn("===== Triad Counting - PSVNet =====");
TimeOut->PutStrLn(Env.GetCmLn());
TimeOut->PutStrFmtLn("Input Time = %f", GetCPUTimeUsage(ts1, ts2));
TimeOut->PutStrFmtLn("Preprocessing Time = %f", GetCPUTimeUsage(ts2, ts3));
TimeOut->PutStrFmtLn("Conversion Time = %f", GetCPUTimeUsage(ts3, ts4));
TimeOut->PutStrFmtLn("Computing Time = %f", GetCPUTimeUsage(ts4, ts5)/nExps);
return 0;
}
//
// Load the schema if required
//
EXPORT_C RDbTableSchema& CDbTableDatabase::SchemaL()
{
RDbTableSchema& schema=Schema();
if (!schema.IsLoaded())
{
schema.Discard();
LoadSchemaL();
schema.Loaded();
}
return schema;
}
MailingListSubscription::MailingListSubscription(
util::RegistryKeyType id/*=0*/
): util::Registrable(id),
Object<MailingListSubscription, MailingListSubscriptionRecord>(
Schema(
FIELD_VALUE_CONSTRUCTOR(Key, id),
FIELD_DEFAULT_CONSTRUCTOR(MailingList),
FIELD_DEFAULT_CONSTRUCTOR(EMailAddress),
FIELD_DEFAULT_CONSTRUCTOR(Parameters)
) )
{}
void Reader::initialize()
{
pdal::Reader::initialize();
Schema& schema = getSchemaRef();
schema = Schema(getDefaultDimensions());
setNumPoints(m_numPoints);
setPointCountType(PointCount_Fixed);
setBounds(m_bounds);
}
MessagesSection::MessagesSection(
util::RegistryKeyType id/*=0*/
): Registrable(id),
Object<MessagesSection, MessagesSectionRecord>(
Schema(
FIELD_VALUE_CONSTRUCTOR(Key, id),
FIELD_DEFAULT_CONSTRUCTOR(Name),
FIELD_VALUE_CONSTRUCTOR(Rank, 0),
FIELD_DEFAULT_CONSTRUCTOR(Code),
FIELD_DEFAULT_CONSTRUCTOR(Color)
) )
{}
void InPlaceReprojection::checkImpedance()
{
Schema& schema = this->getSchemaRef();
schema::index_by_index const& dims = schema.getDimensions().get<schema::index>();
const std::string x_name = getOptions().getValueOrDefault<std::string>("x_dim", "X");
const std::string y_name = getOptions().getValueOrDefault<std::string>("y_dim", "Y");
const std::string z_name = getOptions().getValueOrDefault<std::string>("z_dim", "Z");
schema::index_by_index::const_iterator i;
std::vector<Dimension> new_dimensions;
for (i = dims.begin(); i != dims.end(); ++i)
{
Dimension d(*i);
if (i->getName() == x_name)
{
m_x = *i;
m_x_scale = getOptions().getValueOrDefault<double>("scale_x", i->getNumericScale());
d.setNumericScale(m_x_scale);
m_x_offset = getOptions().getValueOrDefault<double>("offset_x", i->getNumericOffset());
d.setNumericOffset(m_x_offset);
}
if (i->getName() == y_name)
{
m_y = *i;
m_y_scale = getOptions().getValueOrDefault<double>("scale_y", i->getNumericScale());
d.setNumericScale(m_y_scale);
m_y_offset = getOptions().getValueOrDefault<double>("offset_y", i->getNumericOffset());
d.setNumericOffset(m_y_offset);
}
if (i->getName() == z_name)
{
m_z = *i;
m_z_scale = getOptions().getValueOrDefault<double>("scale_z", i->getNumericScale());
d.setNumericScale(m_z_scale);
m_z_offset = getOptions().getValueOrDefault<double>("offset_z", i->getNumericOffset());
d.setNumericOffset(m_z_offset);
}
new_dimensions.push_back(d);
}
schema = Schema(new_dimensions);
return;
}
void CDbTableDatabase::Abandon(TDbLockType aLock)
{
if (aLock>=EDbWriteLock)
{
TSglQueIter<CDbTable> iter(iTables);
for (CDbTable* table;(table=iter++)!=0;)
table->Abandon();
}
if (aLock==EDbSchemaLock)
{
Release(); // ensure no-one is using the old schema
Schema().Discard();
}
}
void PrepareData() {
model_cost[1] = 9;
model_cost[2] = 8;
model_cost[3] = 7;
model_cost[21] = 4;
model_cost[231] = 2;
model_cost[12] = 12;
model_cost[23] = 5;
model_cost[32] = 5;
std::vector<int> attr(3);
attr[0] = attr[1] = attr[2] = 0;
schema = Schema(attr);
RegisterAttrModel(0, new MockModelCreator());
config.allowed_err.resize(3);
}
Schema Schema::from_xml(std::string const& xml, std::string const& xsd)
{
#ifdef PDAL_HAVE_LIBXML2
pdal::schema::Reader reader(xml, xsd);
pdal::Schema schema = reader.getSchema();
return schema;
#else
boost::ignore_unused_variable_warning(xml);
boost::ignore_unused_variable_warning(xsd);
return Schema();
#endif
}
Relation * orderByRelationTwoPass(Relation *relation_ptr, vector<string> &field_name, vector<enum FIELD_TYPE> &field_type,
bool returnRelation) {
Block *block_ptr;
// Every 9 blocks of the relation are sorted
int sorted_blocks = 9;
int num_relation_blocks = relation_ptr->getNumOfBlocks();
int num_groups = (num_relation_blocks - 1) / 9 + 1;
int first_min_index;
vector<int> blocks_used;
Relation *ordered_relation_ptr = NULL;
Schema schema;
if (returnRelation) {
schema = relation_ptr->getSchema();
ordered_relation_ptr = schema_manager.createRelation(relation_ptr->getRelationName() + "_ordered",
Schema(schema.getFieldNames(), schema.getFieldTypes()));
}
// Bring first block of each group in memory
for (int i = 0; i < num_groups; i++) {
block_ptr = mem.getBlock(i);
block_ptr->clear();
relation_ptr->getBlock(i * sorted_blocks, i);
blocks_used.push_back(1);
}
while(1) {
Block *min_tuple_block_ptr = NULL;
int min_tuple_index = -1;
for(int i = 0; i < num_groups; i++) {
block_ptr = mem.getBlock(i);
vector<Tuple> tuples = block_ptr->getTuples();
// Try to get the first unconsidered tuple from this sublist
while ( (first_min_index = getFirstMinTuple(tuples)) == -1) {
// Use 1 more block
blocks_used[i] += 1;
if ( (i == num_groups - 1) && (blocks_used[i] > ((num_relation_blocks - 1) % 9 + 1)))
break;
if (blocks_used[i] > sorted_blocks)
break;
// Bring another block from the sublist into the memory
relation_ptr->getBlock(i * sorted_blocks + blocks_used[i] - 1, i);
tuples = block_ptr->getTuples();
}
// This sublist has been exhausted
if (first_min_index == -1)
continue;
if (isSmaller(field_name, field_type, block_ptr->getTuple(first_min_index),
min_tuple_block_ptr, min_tuple_index)) {
min_tuple_block_ptr = block_ptr;
min_tuple_index = first_min_index;
}
}
// If no min tuple found, break
if (!min_tuple_block_ptr)
break;
Tuple min_tuple = min_tuple_block_ptr->getTuple(min_tuple_index);
if (!returnRelation)
min_tuple.printTuple();
else
appendTupleToRelation(ordered_relation_ptr, 9, min_tuple);
min_tuple_block_ptr->nullTuple(min_tuple_index);
}
return ordered_relation_ptr;
}
NullPointBuffer::NullPointBuffer() : PointBuffer( Schema() )
{}
Schema& AggregateCount::foldinit(libconfig::Config& root, libconfig::Setting& cfg)
{
aggregatecountschema = Schema();
aggregatecountschema.add(CT_LONG);
return aggregatecountschema;
}
ReadQuery(const Reader& reader, const json& j)
: Query(reader, j)
, m_schema(j.count("schema") ?
Schema(j.at("schema")) : m_metadata.outSchema())
{ }
int main(int argc, char* argv[])
{
TEnv Env(argc, argv);
TStr PrefixPath = Env.GetArgs() > 1 ? Env.GetArg(1) : TStr("");
double ts1 = Tick();
TTableContext Context;
TVec<PTable> NodeTblV = TVec<PTable>();
TVec<PTable> EdgeTblV = TVec<PTable>();
Schema NodeSchema = Schema();
Schema EdgeSchema = Schema();
LoadFlickrTables(PrefixPath, Context, NodeTblV, NodeSchema, EdgeTblV, EdgeSchema);
double ts2 = Tick();
int ExpectedSz = 0;
for (TVec<PTable>::TIter it = NodeTblV.BegI(); it < NodeTblV.EndI(); it++) {
PTable Table = *it;
ExpectedSz += Table->GetNumRows();
}
THash<TStr, TInt> Hash(ExpectedSz);
TStrV OriNIdV(ExpectedSz);
MergeNodeTables(NodeTblV, NodeSchema, Hash, OriNIdV);
PTable EdgeTable = MergeEdgeTables(EdgeTblV, EdgeSchema, Hash, Context);
double ts3 = Tick();
TStrV V;
TStrV VE;
VE.Add(EdgeSchema.GetVal(2).GetVal1());
PNEANet Graph = TSnap::ToNetwork<PNEANet>(EdgeTable, EdgeSchema.GetVal(0).GetVal1(), EdgeSchema.GetVal(1).GetVal1(),
V, V, VE, aaLast);
double ts4 = Tick();
//int nExps = 1;
int nExps = 40;
TIntFltH PageRankResults;
for (int i = 0; i < nExps; i++) {
PageRankResults = TIntFltH(ExpectedSz);
#ifdef USE_OPENMP
TSnap::GetWeightedPageRankMP2(Graph, PageRankResults, EdgeSchema.GetVal(2).GetVal1(), 0.849999999999998, 0.0001, 10);
#else
TSnap::GetWeightedPageRank(Graph, PageRankResults, EdgeSchema.GetVal(2).GetVal1(), 0.849999999999998, 0.0001, 10);
#endif
}
double ts5 = Tick();
PSOut ResultOut = TFOut::New(PrefixPath + TStr("page-rank-results.tsv"));
for (TIntFltH::TIter it = PageRankResults.BegI(); it < PageRankResults.EndI(); it++) {
ResultOut->PutStrFmtLn("%s\t%f9", OriNIdV[it.GetKey()].CStr(), it.GetDat().Val);
}
double ts6 = Tick();
bool isPar = false;
#ifdef USE_OPENMP
isPar = true;
#endif
// PSOut FeaturesOut = TFOut::New(PrefixPath + "features.txt");
// FeaturesOut->PutStrFmtLn("Photo %d", PPhotoTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Users %d", PUserTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Tags %d", PTagTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Comments %d", PCommentTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Locations %d", PLocationTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Photo - Owner %d", PPhotoOwnerTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Photo - Comment %d", PPhotoCommentTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Photo - Location %d", PPhotoLocationTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Comment - User %d", PCommentUserTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Comment - User %d", PCommentUserTbl->GetNumRows().Val);
//// FeaturesOut->PutStrFmtLn("Photo - Tagger %d", PPhotoTaggerTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Tagger - Tag %d", PTaggerTagTbl->GetNumRows().Val);
// FeaturesOut->PutStrFmtLn("Total number of nodes = %d", Graph->GetNodes());
// FeaturesOut->PutStrFmtLn("Total number of edges = %d", Graph->GetEdges());
PSOut TimeOut = TFOut::New(PrefixPath + TStr("time.txt"), true);
TimeOut->PutStrFmtLn("Experiment Weighted - %s - %s", PrefixPath.CStr(), (isPar ? "Parallel" : "Sequential"));
TimeOut->PutStrFmtLn("Input Time = %f", GetCPUTimeUsage(ts1, ts2));
TimeOut->PutStrFmtLn("Preprocessing Time = %f", GetCPUTimeUsage(ts2, ts3));
TimeOut->PutStrFmtLn("Conversion Time = %f", GetCPUTimeUsage(ts3, ts4));
TimeOut->PutStrFmtLn("Computing Time = %f", GetCPUTimeUsage(ts4, ts5)/nExps);
TimeOut->PutStrFmtLn("Output Time = %f", GetCPUTimeUsage(ts5, ts6));
return 0;
}
