boost::uint32_t ByteSwap::processBuffer(PointBuffer& dstData, const PointBuffer& srcData) const
{
const Schema& dstSchema = dstData.getSchema();
schema::index_by_index const& dstDims = dstSchema.getDimensions().get<schema::index>();
dstData.setSpatialBounds(srcData.getSpatialBounds());
dstData.copyPointsFast(0, 0, srcData, srcData.getNumPoints());
dstData.setNumPoints(srcData.getNumPoints());
for (boost::uint32_t i = 0; i != dstData.getNumPoints(); ++i)
{
boost::uint8_t* data = dstData.getData(i);
std::size_t position = 0;
for (boost::uint32_t n = 0; n < dstDims.size(); ++n)
{
const Dimension& d = dstSchema.getDimension(n);
std::size_t size = d.getByteSize();
boost::uint8_t* pos = data + position;
SWAP_ENDIANNESS_N(*pos, size);
position = position + size;
}
}
return dstData.getNumPoints();
}
void IteratorBase::copyDatabaseData(PointBuffer& source,
PointBuffer& destination,
Dimension const& dest_dim,
boost::uint32_t source_starting_position,
boost::uint32_t destination_starting_position,
boost::uint32_t howMany)
{
boost::optional<Dimension const&> source_dim = source.getSchema().getDimensionOptional(dest_dim.getName());
if (!source_dim)
{
return;
}
for (boost::uint32_t i = 0; i < howMany; ++i)
{
if (dest_dim.getInterpretation() == source_dim->getInterpretation() &&
dest_dim.getByteSize() == source_dim->getByteSize() &&
pdal::Utils::compare_distance(dest_dim.getNumericScale(), source_dim->getNumericScale()) &&
pdal::Utils::compare_distance(dest_dim.getNumericOffset(), source_dim->getNumericOffset()) &&
dest_dim.getEndianness() == source_dim->getEndianness()
)
{
// FIXME: This test could produce false positives
boost::uint8_t* source_position = source.getData(source_starting_position+i) + source_dim->getByteOffset();
boost::uint8_t* destination_position = destination.getData(destination_starting_position + i) + dest_dim.getByteOffset();
memcpy(destination_position, source_position, source_dim->getByteSize());
}
else
{
PointBuffer::scaleData(source,
destination,
*source_dim,
dest_dim,
source_starting_position + i,
destination_starting_position + i);
}
}
}
/*! bulk copy all the fields from the given point into this object
\param destPointIndex the destination point index to copy the data from
srcPointBuffer at srcPointIndex.
\param srcPointIndex the point index of the PointBuffer to copy from.
\param srcPointBuffer the source PointBuffer to copy from
\param numPoints the number of points to copy
\verbatim embed:rst
.. warning::
This is only legal if the source and destination schemas are
exactly the same. It is up the caller to ensure this is the case.
Additionally, if the schemas are the same :cpp:func:`pdal::Schema::getByteSize()` but of
different compositions, congratulations :)
\endverbatim
*/
inline void copyPointsFast(boost::uint32_t destPointIndex,
boost::uint32_t srcPointIndex,
const PointBuffer& srcPointBuffer,
boost::uint32_t numPoints)
{
const boost::uint8_t* src = srcPointBuffer.getData(srcPointIndex);
boost::uint8_t* dest = getData(destPointIndex);
assert(m_numPoints <= m_capacity);
memcpy(dest, src, m_byteSize * numPoints);
return;
}
boost::uint32_t Predicate::processBuffer(PointBuffer& data, pdal::plang::BufferedInvocation& python) const
{
python.resetArguments();
python.beginChunk(data);
python.execute();
if (!python.hasOutputVariable("Mask"))
{
throw python_error("Mask variable not set in predicate filter function");
}
boost::uint8_t* mask = new boost::uint8_t[data.getNumPoints()];
PointBuffer dstData(data.getSchema(), data.getCapacity());
python.extractResult("Mask", (boost::uint8_t*)mask, data.getNumPoints(), 1, pdal::dimension::RawByte, 1);
boost::uint8_t* dst = dstData.getData(0);
boost::uint8_t* src = data.getData(0);
const Schema& schema = dstData.getSchema();
boost::uint32_t numBytes = schema.getByteSize();
assert(numBytes == data.getSchema().getByteSize());
boost::uint32_t numSrcPoints = data.getNumPoints();
boost::uint32_t count = 0;
for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++)
{
if (mask[srcIndex])
{
memcpy(dst, src, numBytes);
dst += numBytes;
++count;
dstData.setNumPoints(count);
}
src += numBytes;
}
data.copyPointsFast(0, 0, dstData, count);
data.setNumPoints(count);
delete[] mask;
return count;
}
void Writer::PackPointData(PointBuffer const& buffer,
boost::uint8_t** point_data,
boost::uint32_t& point_data_len,
boost::uint32_t& schema_byte_size)
{
// Creates a new buffer that has the ignored dimensions removed from
// it.
schema::index_by_index const& idx = buffer.getSchema().getDimensions().get<schema::index>();
schema_byte_size = 0;
schema::index_by_index::size_type i(0);
for (i = 0; i < idx.size(); ++i)
{
if (! idx[i].isIgnored())
schema_byte_size = schema_byte_size+idx[i].getByteSize();
}
log()->get(logDEBUG) << "Packed schema byte size " << schema_byte_size << std::endl;;
point_data_len = buffer.getNumPoints() * schema_byte_size;
*point_data = new boost::uint8_t[point_data_len];
boost::uint8_t* current_position = *point_data;
for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i)
{
boost::uint8_t* data = buffer.getData(i);
for (boost::uint32_t d = 0; d < idx.size(); ++d)
{
if (! idx[d].isIgnored())
{
memcpy(current_position, data, idx[d].getByteSize());
current_position = current_position+idx[d].getByteSize();
}
data = data + idx[d].getByteSize();
}
}
}
void Diff::checkPoints( StageSequentialIterator* source_iter,
PointBuffer& source_data,
StageSequentialIterator* candidate_iter,
PointBuffer& candidate_data,
ptree& errors)
{
boost::uint32_t i(0);
boost::uint32_t chunk(0);
boost::uint32_t MAX_BADBYTES(20);
boost::uint32_t badbytes(0);
while (!source_iter->atEnd())
{
const boost::uint32_t numSrcRead = source_iter->read(source_data);
const boost::uint32_t numCandidateRead = candidate_iter->read(candidate_data);
if (numSrcRead != numCandidateRead)
{
std::ostringstream oss;
oss << "Unable to read same number of points for chunk number";
errors.put<std::string>("points.error", oss.str());
errors.put<boost::uint32_t>("points.candidate" , numCandidateRead);
errors.put<boost::uint32_t>("points.source" , numSrcRead);
}
chunk++;
pdal::pointbuffer::PointBufferByteSize source_byte_length(0);
pdal::pointbuffer::PointBufferByteSize candidate_byte_length(0);
source_byte_length = static_cast<pdal::pointbuffer::PointBufferByteSize>(source_data.getSchema().getByteSize()) *
static_cast<pdal::pointbuffer::PointBufferByteSize>(source_data.getNumPoints());
candidate_byte_length = static_cast<pdal::pointbuffer::PointBufferByteSize>(candidate_data.getSchema().getByteSize()) *
static_cast<pdal::pointbuffer::PointBufferByteSize>(candidate_data.getNumPoints());
if (source_byte_length != candidate_byte_length)
{
std::ostringstream oss;
oss << "Source byte length != candidate byte length";
errors.put<std::string>("buffer.error", oss.str());
errors.put<boost::uint32_t>("buffer.candidate" , candidate_byte_length);
errors.put<boost::uint32_t>("buffer.source" , source_byte_length);
}
boost::uint8_t* s = source_data.getData(0);
boost::uint8_t* c = candidate_data.getData(0);
for (boost::uint32_t p = 0; p < std::min(source_byte_length, candidate_byte_length); ++p)
{
if (s[p] != c[p])
{
std::ostringstream oss;
oss << "Byte number " << p << " is not equal for source and candidate";
errors.put<std::string>("data.error", oss.str());
badbytes++;
}
}
if (badbytes > MAX_BADBYTES )
break;
}
}
bool Writer::WriteBlock(PointBuffer const& buffer)
{
boost::uint8_t* point_data;
boost::uint32_t point_data_length;
boost::uint32_t schema_byte_size;
bool pack = getOptions().getValueOrDefault<bool>("pack_ignored_fields", true);
if (pack)
PackPointData(buffer, &point_data, point_data_length, schema_byte_size);
else
{
point_data = buffer.getData(0);
point_data_length = buffer.getSchema().getByteSize() * buffer.getNumPoints();
}
bool doKDTree = getOptions().getValueOrDefault<bool>("kdtreeify", true);
if (doKDTree)
{
// kdtreeify(buffer, &point_data, point_data_length, schema_byte_size);
}
std::string block_table = getOptions().getValueOrThrow<std::string>("block_table");
// // Pluck the block id out of the first point in the buffer
pdal::Schema const& schema = buffer.getSchema();
Dimension const& blockDim = schema.getDimension("BlockID");
m_block_id = buffer.getField<boost::int32_t>(blockDim, 0);
m_obj_id = getOptions().getValueOrThrow<boost::int32_t>("pc_id");
m_num_points = static_cast<boost::int64_t>(buffer.getNumPoints());
// if (m_type == DATABASE_POSTGRESQL)
// {
// std::string cloud_column = getOptions().getValueOrDefault<std::string>("cloud_column", "id");
// bool is3d = getOptions().getValueOrDefault<bool>("is3d", false);
//
//
// std::vector<boost::uint8_t> block_data;
// for (boost::uint32_t i = 0; i < point_data_length; ++i)
// {
// block_data.push_back(point_data[i]);
// }
//
// if (pack)
// delete point_data;
// m_block_bytes.str("");
// Utils::binary_to_hex_stream(point_data, m_block_bytes, 0, point_data_length);
// m_block_data = m_block_bytes.str();
// //std::cout << "hex: " << hex.substr(0, 30) << std::endl;
// m_srid = getOptions().getValueOrDefault<boost::uint32_t>("srid", 4326);
//
// boost::uint32_t precision(9);
// pdal::Bounds<double> bounds = buffer.calculateBounds(3);
// // m_extent.str("");
// m_extent = bounds.toWKT(precision); // polygons are only 2d, not cubes
// // m_bbox.str("");
// m_bbox = bounds.toBox(precision, 3);
// log()->get(logDEBUG) << "extent: " << m_extent << std::endl;
// log()->get(logDEBUG) << "bbox: " << m_bbox << std::endl;
//
// if (!m_block_statement)
// {
// // m_block_statement = (m_session->prepare << m_block_insert_query.str(), \
// // ::soci::use(m_obj_id, "obj_id"), \
// // ::soci::use(m_block_id, "block_id"), \
// // ::soci::use(m_num_points, "num_points"), \
// // ::soci::use(m_block_bytes.str(),"hex"), \
// // ::soci::use(m_extent.str(), "extent"), \
// // ::soci::use(m_srid, "srid"), \
// // ::soci::use(m_bbox.str(), "bbox"));
// m_block_statement = new ::soci::statement(*m_session);
//
// m_block_statement->exchange(::soci::use(m_obj_id, "obj_id"));
// m_block_statement->exchange(::soci::use(m_block_id, "block_id"));
// m_block_statement->exchange(::soci::use(m_num_points, "num_points"));
// m_block_statement->exchange(::soci::use(m_block_data,"hex"));
// m_block_statement->exchange(::soci::use(m_extent, "extent"));
// m_block_statement->exchange(::soci::use(m_srid, "srid"));
// m_block_statement->exchange(::soci::use(m_bbox, "bbox"));
// m_block_statement->alloc();
// m_block_statement->prepare(m_block_insert_query.str());
// m_block_statement->define_and_bind();
//
// }
// // ::soci::statement st = (m_session->prepare << m_block_insert_query.str(), \
// // ::soci::use(m_obj_id, "obj_id"), \
// // ::soci::use(m_block_id, "block_id"), \
// // ::soci::use(m_num_points, "num_points"), \
// // ::soci::use(m_block_bytes.str(),"hex"), \
// // ::soci::use(m_extent.str(), "extent"), \
// // ::soci::use(m_srid, "srid"), \
// // ::soci::use(m_bbox.str(), "bbox"));
// try
// {
// m_block_statement->execute(true);
// }
// catch (std::exception const& e)
// {
// std::ostringstream oss;
// oss << "Insert query failed with error '" << e.what() << "'";
// m_session->rollback();
// throw pdal_error(oss.str());
// }
//
// }
return true;
}
bool Writer::WriteBlock(PointBuffer const& buffer)
{
bool bUsePartition = m_block_table_partition_column.size() != 0;
// Pluck the block id out of the first point in the buffer
pdal::Schema const& schema = buffer.getSchema();
Dimension const& blockDim = schema.getDimension("BlockID");
boost::int32_t block_id = buffer.getField<boost::int32_t>(blockDim, 0);
std::ostringstream oss;
std::ostringstream partition;
if (bUsePartition)
{
partition << "," << m_block_table_partition_column;
}
oss << "INSERT INTO "<< m_block_table_name <<
"(OBJ_ID, BLK_ID, NUM_POINTS, POINTS, "
"PCBLK_MIN_RES, BLK_EXTENT, PCBLK_MAX_RES, NUM_UNSORTED_POINTS, PT_SORT_DIM";
if (bUsePartition)
oss << partition.str();
oss << ") "
"VALUES ( :1, :2, :3, :4, 1, mdsys.sdo_geometry(:5, :6, null,:7, :8)"
", 1, 0, 1";
if (bUsePartition)
oss << ", :9";
oss <<")";
// TODO: If gotdata == false below, this memory probably leaks --mloskot
OCILobLocator** locator =(OCILobLocator**) VSIMalloc(sizeof(OCILobLocator*) * 1);
Statement statement = Statement(m_connection->CreateStatement(oss.str().c_str()));
long* p_pc_id = (long*) malloc(1 * sizeof(long));
p_pc_id[0] = m_pc_id;
long* p_result_id = (long*) malloc(1 * sizeof(long));
p_result_id[0] = (long)block_id;
long* p_num_points = (long*) malloc(1 * sizeof(long));
p_num_points[0] = (long)buffer.getNumPoints();
// std::cout << "point count on write: " << buffer.getNumPoints() << std::endl;
// :1
statement->Bind(&m_pc_id);
// :2
statement->Bind(p_result_id);
// :3
statement->Bind(p_num_points);
// :4
statement->Define(locator, 1);
// std::vector<liblas::uint8_t> data;
// bool gotdata = GetResultData(result, reader, data, 3);
// if (! gotdata) throw std::runtime_error("unable to fetch point data byte array");
// boost::uint8_t* point_data = buffer.getData(0);
boost::uint8_t* point_data;
boost::uint32_t point_data_length;
boost::uint32_t schema_byte_size;
bool pack = getOptions().getValueOrDefault<bool>("pack_ignored_fields", true);
if (pack)
PackPointData(buffer, &point_data, point_data_length, schema_byte_size);
else
{
point_data = buffer.getData(0);
point_data_length = buffer.getSchema().getByteSize() * buffer.getNumPoints();
}
// statement->Bind((char*)point_data,(long)(buffer.getSchema().getByteSize()*buffer.getNumPoints()));
statement->Bind((char*)point_data,(long)(point_data_length));
// :5
long* p_gtype = (long*) malloc(1 * sizeof(long));
p_gtype[0] = m_gtype;
statement->Bind(p_gtype);
// :6
long* p_srid = 0;
if (m_srid != 0)
{
p_srid = (long*) malloc(1 * sizeof(long));
p_srid[0] = m_srid;
}
statement->Bind(p_srid);
// :7
OCIArray* sdo_elem_info=0;
m_connection->CreateType(&sdo_elem_info, m_connection->GetElemInfoType());
SetElements(statement, sdo_elem_info);
statement->Bind(&sdo_elem_info, m_connection->GetElemInfoType());
// :8
OCIArray* sdo_ordinates=0;
m_connection->CreateType(&sdo_ordinates, m_connection->GetOrdinateType());
// x0, x1, y0, y1, z0, z1, bUse3d
pdal::Bounds<double> bounds = CalculateBounds(buffer);
SetOrdinates(statement, sdo_ordinates, bounds);
statement->Bind(&sdo_ordinates, m_connection->GetOrdinateType());
// :9
long* p_partition_d = 0;
if (bUsePartition)
{
p_partition_d = (long*) malloc(1 * sizeof(long));
p_partition_d[0] = m_block_table_partition_value;
statement->Bind(p_partition_d);
}
try
{
statement->Execute();
}
catch (std::runtime_error const& e)
{
std::ostringstream oss;
oss << "Failed to insert block # into '" << m_block_table_name << "' table. Does the table exist? " << std::endl << e.what() << std::endl;
throw std::runtime_error(oss.str());
}
oss.str("");
OWStatement::Free(locator, 1);
if (p_pc_id != 0) free(p_pc_id);
if (p_result_id != 0) free(p_result_id);
if (p_num_points != 0) free(p_num_points);
if (p_gtype != 0) free(p_gtype);
if (p_srid != 0) free(p_srid);
if (p_partition_d != 0) free(p_partition_d);
m_connection->DestroyType(&sdo_elem_info);
m_connection->DestroyType(&sdo_ordinates);
return true;
}
void Writer::PackPointData(PointBuffer const& buffer,
boost::uint8_t** point_data,
boost::uint32_t& point_data_len,
boost::uint32_t& schema_byte_size)
{
// Creates a new buffer that has the ignored dimensions removed from
// it.
schema::index_by_index const& idx = buffer.getSchema().getDimensions().get<schema::index>();
schema_byte_size = 0;
schema::index_by_index::size_type i(0);
for (i = 0; i < idx.size(); ++i)
{
if (! idx[i].isIgnored())
schema_byte_size = schema_byte_size+idx[i].getByteSize();
}
log()->get(logDEBUG) << "Packed schema byte size " << schema_byte_size;
point_data_len = buffer.getNumPoints() * schema_byte_size;
*point_data = new boost::uint8_t[point_data_len];
boost::uint8_t* current_position = *point_data;
for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i)
{
boost::uint8_t* data = buffer.getData(i);
for (boost::uint32_t d = 0; d < idx.size(); ++d)
{
if (! idx[d].isIgnored())
{
memcpy(current_position, data, idx[d].getByteSize());
current_position = current_position+idx[d].getByteSize();
}
data = data + idx[d].getByteSize();
}
}
// Create a vector of two element vectors that states how long of a byte
// run to copy for the point based on whether or not the ignored/used
// flag of the dimension switches. This is to a) eliminate panning through
// the dimension multi_index repeatedly per point, and to b) eliminate
// tons of small memcpy in exchange for larger ones.
// std::vector< std::vector< boost::uint32_t> > runs;
// // bool switched = idx[0].isIgnored(); // start at with the first dimension
// for (boost::uint32_t d = 0; d < idx.size(); ++d)
// {
// std::vector<boost::uint32_t> t;
// if ( idx[d].isIgnored())
// {
// t.push_back(0);
// t.push_back(idx[d].getByteSize());
// }
// else
// {
// t.push_back(1);
// t.push_back(idx[d].getByteSize());
// }
//
// runs.push_back(t);
// }
//
// for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i)
// {
// boost::uint8_t* data = buffer.getData(i);
// for (std::vector< std::vector<boost::uint32_t> >::size_type d=0; d < runs.size(); d++)
// // for (boost::uint32_t d = 0; d < idx.size(); ++d)
// {
// boost::uint32_t isUsed = runs[d][0];
// boost::uint32_t byteSize = runs[d][1];
// if (isUsed != 0)
// {
// memcpy(current_position, data, byteSize);
// current_position = current_position+byteSize;
// }
// data = data + byteSize;
//
// }
// }
}
