static void end_fileobject(user_data_t& user_data) {
if (user_data.fileobject_repository_name == "" ||
user_data.fileobject_filename == "") {
// insufficiently defined
return;
}
// hashdigest type
if (user_data.byte_run_hashdigest_type != "" &&
user_data.byte_run_hashdigest_type != "MD5") {
std::cerr << "dfxml_hashdigest_reader: Unexpected hashdigest type for fileobject: '"
<< user_data.fileobject_hashdigest_type << "'.\n";
return;
}
// file hashdigest
std::string binary_hash = lmdb_helper::hex_to_binary_hash(
user_data.fileobject_hashdigest);
// file size
uint64_t filesize = std::atol(user_data.fileobject_filesize.c_str());
// put source_data together
lmdb_source_data_t source_data(user_data.fileobject_repository_name,
user_data.fileobject_filename,
filesize,
binary_hash);
// consume source_data
user_data.dfxml_consumer->end_fileobject(source_data);
}
void injective_join(injective_join_index& target,
TargetGraph& target_graph,
injective_join_index& source,
SourceGraph& source_graph,
JoinOp joinop) {
// build up the exchange structure.
// move right vertex data to left
std::vector<
std::vector<
std::pair<size_t, typename SourceGraph::vertex_data_type> > >
source_data(rmi.numprocs());
for (size_t i = 0; i < source.opposing_join_proc.size(); ++i) {
if (source_graph.l_vertex(i).owned()) {
procid_t target_proc = source.opposing_join_proc[i];
if (target_proc >= 0 && target_proc < rmi.numprocs()) {
source_data[target_proc].push_back(
std::make_pair(source.vtx_to_key[i],
source_graph.l_vertex(i).data()));
}
}
}
// exchange
rmi.all_to_all(source_data);
// ok. now join against left
#ifdef _OPENMP
#pragma omp parallel for
#endif
for (size_t p = 0;p < source_data.size(); ++p) {
for (size_t i = 0;i < source_data[p].size(); ++i) {
// find the target vertex with the matching key
hopscotch_map<size_t, vertex_id_type>::const_iterator iter =
target.key_to_vtx.find(source_data[p][i].first);
ASSERT_TRUE(iter != target.key_to_vtx.end());
// found it!
typename TargetGraph::local_vertex_type
lvtx = target_graph.l_vertex(iter->second);
typename TargetGraph::vertex_type vtx(lvtx);
joinop(vtx, source_data[p][i].second);
}
}
target_graph.synchronize();
}
static void end_byte_run(user_data_t& user_data) {
// pull together byte_run fields for the hashdb element
// hashdigest
std::string binary_hash = lmdb_helper::hex_to_binary_hash(
user_data.byte_run_hashdigest);
// reject invalid input
if (binary_hash == "") {
std::cout << "Entry ignored for invalid hash '"
<< user_data.byte_run_hashdigest << "'\n";
return;
}
// file_offset
uint64_t file_offset;
file_offset = std::atol(user_data.byte_run_file_offset.c_str());
// hash_block_size
uint32_t hash_block_size = std::atol(user_data.byte_run_len.c_str());
// hashdigest type
if (user_data.byte_run_hashdigest_type != "" &&
user_data.byte_run_hashdigest_type != "MD5") {
std::cerr << "dfxml_hashdigest_reader: Unexpected hashdigest type for byte_run: '"
<< user_data.byte_run_hashdigest_type << "'.\n";
return;
}
// create the source data
lmdb_source_data_t source_data(user_data.fileobject_repository_name,
user_data.fileobject_filename, 0, "");
// call the hash consumer
user_data.dfxml_consumer->end_byte_run(binary_hash,
file_offset,
hash_block_size,
source_data,
user_data.byte_run_hash_label);
}
int Diff::execute()
{
Options sourceOptions;
{
sourceOptions.add<std::string>("filename", m_sourceFile);
sourceOptions.add<bool>("debug", isDebug());
sourceOptions.add<boost::uint32_t>("verbose", getVerboseLevel());
}
boost::scoped_ptr<Stage> source(AppSupport::makeReader(sourceOptions));
source->initialize();
boost::uint32_t chunkSize(source->getNumPoints());
if (m_chunkSize)
chunkSize = m_chunkSize;
PointBuffer source_data(source->getSchema(), chunkSize);
boost::scoped_ptr<StageSequentialIterator> source_iter(source->createSequentialIterator(source_data));
ptree errors;
Options candidateOptions;
{
candidateOptions.add<std::string>("filename", m_candidateFile);
candidateOptions.add<bool>("debug", isDebug());
candidateOptions.add<boost::uint32_t>("verbose", getVerboseLevel());
}
boost::scoped_ptr<Stage> candidate(AppSupport::makeReader(candidateOptions));
candidate->initialize();
PointBuffer candidate_data(candidate->getSchema(), chunkSize);
boost::scoped_ptr<StageSequentialIterator> candidate_iter(candidate->createSequentialIterator(candidate_data));
if (candidate->getNumPoints() != source->getNumPoints())
{
std::ostringstream oss;
oss << "Source and candidate files do not have the same point count";
errors.put<std::string>("count.error", oss.str());
errors.put<boost::uint32_t>("count.candidate" , candidate->getNumPoints());
errors.put<boost::uint32_t>("count.source" , source->getNumPoints());
}
pdal::Metadata source_metadata = source->collectMetadata();
pdal::Metadata candidate_metadata = candidate->collectMetadata();
if (source_metadata != candidate_metadata)
{
std::ostringstream oss;
oss << "Source and candidate files do not have the same metadata count";
errors.put<std::string>("metadata.error", oss.str());
errors.put_child("metadata.source", source_metadata.toPTree());
errors.put_child("metadata.candidate", candidate_metadata.toPTree());
}
Schema const& candidate_schema = candidate_data.getSchema();
Schema const& source_schema = source_data.getSchema();
if (! ( candidate_schema == source_schema))
{
std::ostringstream oss;
oss << "Source and candidate files do not have the same schema";
errors.put<std::string>("schema.error", oss.str());
errors.put_child("schema.source", source_schema.toPTree());
errors.put_child("schema.candidate", candidate_schema.toPTree());
}
if (errors.size())
{
write_json(std::cout, errors);
return 1;
} else
{
// If we made it this far with no errors, now we'll
// check the points.
checkPoints(source_iter.get(),
source_data,
candidate_iter.get(),
candidate_data,
errors);
if (errors.size())
{
write_json(std::cout, errors);
return 1;
}
}
return 0;
}
int Delta::execute()
{
Options sourceOptions;
{
sourceOptions.add<std::string>("filename", m_sourceFile);
sourceOptions.add<bool>("debug", isDebug());
sourceOptions.add<boost::uint32_t>("verbose", getVerboseLevel());
}
Stage* source = AppSupport::makeReader(sourceOptions);
source->initialize();
boost::uint32_t totalPointCount(source->getNumPoints());
PointBuffer source_data(source->getSchema(), totalPointCount);
StageSequentialIterator* source_iter = source->createSequentialIterator(source_data);
boost::uint32_t numRead = source_iter->read(source_data);
assert(numRead == source_data.getNumPoints());
delete source_iter;
delete source;
Options candidateOptions;
{
candidateOptions.add<std::string>("filename", m_candidateFile);
candidateOptions.add<bool>("debug", isDebug());
candidateOptions.add<boost::uint32_t>("verbose", getVerboseLevel());
}
Stage* candidate = AppSupport::makeReader(candidateOptions);
candidate->initialize();
IndexedPointBuffer candidate_data(candidate->getSchema(), totalPointCount);
StageSequentialIterator* candidate_iter = candidate->createSequentialIterator(candidate_data);
numRead = candidate_iter->read(candidate_data);
assert(numRead == candidate_data.getNumPoints());
delete candidate_iter;
if (source_data.getNumPoints() != candidate_data.getNumPoints())
{
std::cerr << "Source and candidate files do not have the same point count, testing each source point only!" << std::endl;
}
// m_summary_x(xd);
// m_summary_y(yd);
// m_summary_z(zd);
if (m_outputFileName.size())
{
m_outputStream = FileUtils::createFile(m_outputFileName);
}
candidate_data.build(m_3d);
boost::uint32_t count(std::min(source_data.getNumPoints(), candidate_data.getNumPoints()));
boost::scoped_ptr<std::map<Point, Point> > points(cumulatePoints(source_data, candidate_data));
if (m_OutputDetail)
{
outputDetail(source_data, candidate_data, points.get());
return 0;
}
std::map<Point, Point>::const_iterator i;
for(i = points->begin(); i != points->end(); ++i)
{
Point const& s = i->first;
Point const& c = i->second;
double xd = s.x - c.x;
double yd = s.y - c.y;
double zd = s.z - c.z;
m_summary_x(xd);
m_summary_y(yd);
m_summary_z(zd);
}
std::string headline("------------------------------------------------------------------------------------------");
std::cout << headline << std::endl;
std::cout << " Delta summary for source '" << m_sourceFile << "' and candidate '" << m_candidateFile <<"'" << std::endl;
std::cout << headline << std::endl;
std::cout << std::endl;
std::string thead("----------- --------------- --------------- --------------");
std::cout << thead << std::endl;
std::cout << " Dimension X Y Z " << std::endl;
std::cout << thead << std::endl;
boost::format fmt("%.4f");
double sminx = (boost::accumulators::min)(m_summary_x);
double sminy = (boost::accumulators::min)(m_summary_y);
double sminz = (boost::accumulators::min)(m_summary_z);
double smaxx = (boost::accumulators::max)(m_summary_x);
double smaxy = (boost::accumulators::max)(m_summary_y);
double smaxz = (boost::accumulators::max)(m_summary_z);
double smeanx = (boost::accumulators::mean)(m_summary_x);
double smeany = (boost::accumulators::mean)(m_summary_y);
double smeanz = (boost::accumulators::mean)(m_summary_z);
std::cout << " Min " << fmt % sminx << " " << fmt % sminy << " " << fmt % sminz<<std::endl;
std::cout << " Min " << fmt % smaxx << " " << fmt % smaxy << " " << fmt % smaxz<<std::endl;
std::cout << " Mean " << fmt % smeanx << " " << fmt % smeany << " " << fmt % smeanz<<std::endl;
std::cout << thead << std::endl;
return 0;
}
