boost::uint32_t Crop::readBufferImpl(PointBuffer& dstData) { // The client has asked us for dstData.getCapacity() points. // We will read from our previous stage until we get that amount (or // until the previous stage runs out of points). boost::uint32_t numPointsNeeded = dstData.getCapacity(); assert(dstData.getNumPoints() == 0); while (numPointsNeeded > 0) { if (getPrevIterator().atEnd()) break; // set up buffer to be filled by prev stage PointBuffer srcData(dstData.getSchema(), numPointsNeeded); // read from prev stage const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData); assert(numSrcPointsRead == srcData.getNumPoints()); assert(numSrcPointsRead <= numPointsNeeded); // we got no data, and there is no more to get -- exit the loop if (numSrcPointsRead == 0) break; // copy points from src (prev stage) into dst (our stage), // based on the CropFilter's rules (i.e. its bounds) const boost::uint32_t numPointsProcessed = m_cropFilter.processBuffer(dstData, srcData); numPointsNeeded -= numPointsProcessed; } const boost::uint32_t numPointsAchieved = dstData.getNumPoints(); return numPointsAchieved; }
boost::uint32_t DecimationFilterSequentialIterator::readBufferImpl(PointBuffer& dstData) { // The client has asked us for dstData.getCapacity() points. // We will read from our previous stage until we get that amount (or // until the previous stage runs out of points). boost::uint32_t numPointsNeeded = dstData.getCapacity(); assert(dstData.getNumPoints() == 0); // set up buffer to be filled by prev stage PointBuffer srcData(dstData.getSchemaLayout(), numPointsNeeded); while (numPointsNeeded > 0) { // read from prev stage const boost::uint64_t srcStartIndex = getPrevIterator().getIndex(); const boost::uint32_t numSrcPointsRead = getPrevIterator().read(srcData); assert(numSrcPointsRead <= srcData.getNumPoints()); //assert(numSrcPointsRead <= numPointsNeeded); // we got no data, and there is no more to get -- exit the loop if (numSrcPointsRead == 0) break; // copy points from src (prev stage) into dst (our stage), // based on the CropFilter's rules (i.e. its bounds) const boost::uint32_t numPointsAdded = m_filter.processBuffer(dstData, srcData, srcStartIndex); numPointsNeeded -= numPointsAdded; //printf(".");fflush(stdout); } const boost::uint32_t numPointsAchieved = dstData.getNumPoints(); return numPointsAchieved; }
boost::uint32_t ByteSwap::processBuffer(PointBuffer& dstData, const PointBuffer& srcData) const { const Schema& dstSchema = dstData.getSchema(); schema::index_by_index const& dstDims = dstSchema.getDimensions().get<schema::index>(); dstData.setSpatialBounds(srcData.getSpatialBounds()); dstData.copyPointsFast(0, 0, srcData, srcData.getNumPoints()); dstData.setNumPoints(srcData.getNumPoints()); for (boost::uint32_t i = 0; i != dstData.getNumPoints(); ++i) { boost::uint8_t* data = dstData.getData(i); std::size_t position = 0; for (boost::uint32_t n = 0; n < dstDims.size(); ++n) { const Dimension& d = dstSchema.getDimension(n); std::size_t size = d.getByteSize(); boost::uint8_t* pos = data + position; SWAP_ENDIANNESS_N(*pos, size); position = position + size; } } return dstData.getNumPoints(); }
boost::uint32_t Predicate::processBuffer(PointBuffer& data, pdal::plang::BufferedInvocation& python) const { python.resetArguments(); python.beginChunk(data); python.execute(); if (!python.hasOutputVariable("Mask")) { throw python_error("Mask variable not set in predicate filter function"); } boost::uint8_t* mask = new boost::uint8_t[data.getNumPoints()]; PointBuffer dstData(data.getSchema(), data.getCapacity()); python.extractResult("Mask", (boost::uint8_t*)mask, data.getNumPoints(), 1, pdal::dimension::RawByte, 1); boost::uint8_t* dst = dstData.getData(0); boost::uint8_t* src = data.getData(0); const Schema& schema = dstData.getSchema(); boost::uint32_t numBytes = schema.getByteSize(); assert(numBytes == data.getSchema().getByteSize()); boost::uint32_t numSrcPoints = data.getNumPoints(); boost::uint32_t count = 0; for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++) { if (mask[srcIndex]) { memcpy(dst, src, numBytes); dst += numBytes; ++count; dstData.setNumPoints(count); } src += numBytes; } data.copyPointsFast(0, 0, dstData, count); data.setNumPoints(count); delete[] mask; return count; }
void InPlaceReprojection::processBuffer(PointBuffer& data) const { const boost::uint32_t numPoints = data.getNumPoints(); const Schema& schema = this->getSchema(); Dimension const& d_x = schema.getDimension(getOptions().getValueOrDefault<std::string>("x_dim", "X")); Dimension const& d_y = schema.getDimension(getOptions().getValueOrDefault<std::string>("y_dim", "Y")); Dimension const& d_z = schema.getDimension(getOptions().getValueOrDefault<std::string>("z_dim", "Z")); for (boost::uint32_t pointIndex=0; pointIndex<numPoints; pointIndex++) { double x = getScaledValue(data, m_x, pointIndex); double y = getScaledValue(data, m_y, pointIndex); double z = getScaledValue(data, m_z, pointIndex); // std::cout << "input: " << x << " y: " << y << " z: " << z << std::endl; this->transform(x,y,z); // std::cout << "output: " << x << " y: " << y << " z: " << z << std::endl; setScaledValue(data, x, d_x, pointIndex); setScaledValue(data, y, d_y, pointIndex); setScaledValue(data, z, d_z, pointIndex); // std::cout << "set: " << getScaledValue(data, d_x, pointIndex, indexX) // << " y: " << getScaledValue(data, d_y, pointIndex, indexY) // << " z: " << getScaledValue(data, d_z, pointIndex, indexZ) << std::endl; data.setNumPoints(pointIndex+1); } return; }
boost::uint32_t Mosaic::readBufferImpl(PointBuffer& destData) { boost::uint32_t totalNumPointsToRead = destData.getCapacity(); boost::uint32_t totalNumPointsRead = 0; boost::uint32_t destPointIndex = 0; // for each stage, we read as many points as we can while (totalNumPointsRead < totalNumPointsToRead) { assert(m_iteratorIndex < m_prevIterators.size()); m_prevIterator = m_prevIterators[m_iteratorIndex]; // read as much as we can into temp buffer PointBuffer tmp(destData.getSchema(), totalNumPointsToRead-totalNumPointsRead); boost::uint32_t numRead = m_prevIterator->read(tmp); totalNumPointsRead += numRead; // concat the temp buffer on to end of real dest buffer destData.copyPointsFast(destPointIndex, 0, tmp, numRead); destPointIndex += numRead; destData.setNumPoints(destData.getNumPoints() + numRead); if (m_prevIterator->atEnd()) { ++m_iteratorIndex; } if (m_iteratorIndex == m_prevIterators.size()) { break; } } return totalNumPointsRead; }
void Reprojection::processBuffer(PointBuffer& data) const { const boost::uint32_t numPoints = data.getNumPoints(); const Schema& schema = data.getSchema(); Dimension const& dimX = schema.getDimension("X"); Dimension const& dimY = schema.getDimension("Y"); Dimension const& dimZ = schema.getDimension("Z"); for (boost::uint32_t pointIndex=0; pointIndex<numPoints; pointIndex++) { double x = data.getField<double>(dimX, pointIndex); double y = data.getField<double>(dimY, pointIndex); double z = data.getField<double>(dimZ, pointIndex); this->transform(x,y,z); data.setField<double>(dimX, pointIndex, x); data.setField<double>(dimY, pointIndex, y); data.setField<double>(dimZ, pointIndex, z); data.setNumPoints(pointIndex+1); } return; }
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer) { Schema const& user_schema = user_buffer.getSchema(); schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>(); boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints(); if (numUserSpace < 0) throw pdal_error("We ran out of space!"); boost::int32_t numOraclePoints = m_active_buffer->getNumPoints() - m_buffer_position; schema::index_by_index::size_type i(0); for (i = 0; i < idx.size(); ++i) { copyDatabaseData(*m_active_buffer, user_buffer, idx[i], m_buffer_position, user_buffer.getNumPoints(), (std::min)(numOraclePoints,numUserSpace)); } bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false); if (bSetPointSourceId) { Dimension const* point_source_field = &(user_buffer.getSchema().getDimensionOptional("PointSourceId").get()); if (point_source_field) { for (boost::int32_t i = 0; i < numUserSpace; ++i) { if (i < 0) throw sqlite_driver_error("point_source_field point index is less than 0!"); user_buffer.setField(*point_source_field, i, m_active_cloud_id); } } } if (numOraclePoints > numUserSpace) m_buffer_position = m_buffer_position + numUserSpace; else if (numOraclePoints < numUserSpace) m_buffer_position = 0; boost::uint32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints); user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints()); }
boost::uint32_t Predicate::readBufferImpl(PointBuffer& dstData) { // read in a full block of points const boost::uint32_t numRead = getPrevIterator().read(dstData); if (numRead > 0) { m_numPointsProcessed = dstData.getNumPoints(); // copies the points as they pass in-place m_predicateFilter.processBuffer(dstData, *m_pythonMethod); } m_numPointsPassed = dstData.getNumPoints(); return dstData.getNumPoints(); }
boost::uint32_t Writer::writeBuffer(const PointBuffer& buffer) { boost::uint32_t numPoints = buffer.getNumPoints(); WriteBlock(buffer); return numPoints; }
void Writer::PackPointData(PointBuffer const& buffer, boost::uint8_t** point_data, boost::uint32_t& point_data_len, boost::uint32_t& schema_byte_size) { // Creates a new buffer that has the ignored dimensions removed from // it. schema::index_by_index const& idx = buffer.getSchema().getDimensions().get<schema::index>(); schema_byte_size = 0; schema::index_by_index::size_type i(0); for (i = 0; i < idx.size(); ++i) { if (! idx[i].isIgnored()) schema_byte_size = schema_byte_size+idx[i].getByteSize(); } log()->get(logDEBUG) << "Packed schema byte size " << schema_byte_size << std::endl;; point_data_len = buffer.getNumPoints() * schema_byte_size; *point_data = new boost::uint8_t[point_data_len]; boost::uint8_t* current_position = *point_data; for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i) { boost::uint8_t* data = buffer.getData(i); for (boost::uint32_t d = 0; d < idx.size(); ++d) { if (! idx[d].isIgnored()) { memcpy(current_position, data, idx[d].getByteSize()); current_position = current_position+idx[d].getByteSize(); } data = data + idx[d].getByteSize(); } } }
Reader::Reader(const Options& options, const PointBuffer& buffer) : pdal::Reader(options) , m_buffer(buffer) { setNumPoints(buffer.getNumPoints()); setBounds(buffer.getSpatialBounds()); setSchema(buffer.getSchema()); setPointCountType(PointCount_Fixed); return; }
void PDALtoPCD(PointBuffer& data, CloudT &cloud) { #ifdef PDAL_HAVE_PCL typedef typename pcl::traits::fieldList<typename CloudT::PointType>::type FieldList; const pdal::Schema &buffer_schema = data.getSchema(); cloud.width = data.getNumPoints(); cloud.height = 1; // unorganized point cloud cloud.is_dense = false; cloud.points.resize(cloud.width); const pdal::Dimension &dX = buffer_schema.getDimension("X"); const pdal::Dimension &dY = buffer_schema.getDimension("Y"); const pdal::Dimension &dZ = buffer_schema.getDimension("Z"); if (pcl::traits::has_xyz<typename CloudT::PointType>::value) { for (size_t i = 0; i < cloud.points.size(); ++i) { float xd = data.getField<int32_t>(dX, i) * dX.getNumericScale(); float yd = data.getField<int32_t>(dY, i) * dY.getNumericScale(); float zd = data.getField<int32_t>(dZ, i) * dZ.getNumericScale(); typename CloudT::PointType p = cloud.points[i]; p.x = xd; p.y = yd; p.z = zd; cloud.points[i] = p; } } boost::optional<pdal::Dimension const &> dI = buffer_schema.getDimensionOptional("Intensity"); if (pcl::traits::has_intensity<typename CloudT::PointType>::value && dI) { for (size_t i = 0; i < cloud.points.size(); ++i) { boost::int32_t vi = data.getField<boost::int32_t>(*dI, i); float vd = dI->applyScaling(vi); typename CloudT::PointType p = cloud.points[i]; pcl::for_each_type<FieldList> (pcl::SetIfFieldExists<typename CloudT::PointType, float> (p, "intensity", vd)); cloud.points[i] = p; } } #endif }
PointBuffer* makeTestBuffer() { Dimension d1("Classification", dimension::UnsignedInteger, 1); Dimension d2("X", dimension::SignedInteger, 4); Dimension d3("Y", dimension::Float, 8); Schema schema; schema.appendDimension(d1); schema.appendDimension(d2); schema.appendDimension(d3); std::size_t offX = schema.getDimension(0).getByteOffset(); BOOST_CHECK(offX==0); std::size_t offY = schema.getDimension(1).getByteOffset(); BOOST_CHECK(offY==1); std::size_t offZ = schema.getDimension(2).getByteOffset(); BOOST_CHECK(offZ==5); boost::uint32_t capacity = 17; PointBuffer* data = new PointBuffer(schema, capacity); BOOST_CHECK(data->getCapacity() == capacity); Dimension const& dimC = data->getSchema().getDimension("Classification"); Dimension const& dimX = data->getSchema().getDimension("X"); Dimension const& dimY = data->getSchema().getDimension("Y"); // write the data into the buffer for (boost::uint32_t i=0; i<data->getCapacity(); i++) { const boost::uint8_t x = static_cast<boost::uint8_t>(i)+1; const boost::int32_t y = i*10; const double z = i * 100; data->setField(dimC, i, x); data->setField(dimX, i, y); data->setField(dimY, i, z); data->setNumPoints(i+1); } BOOST_CHECK(data->getCapacity() ==17); BOOST_CHECK(data->getNumPoints() ==17); return data; }
boost::uint32_t Writer::writeBuffer(const PointBuffer& data) { const Schema& schema = data.getSchema(); std::string z_name = getOptions().getValueOrDefault<std::string>("Z", "Z"); pdal::Dimension const& dimX = schema.getDimension("X"); pdal::Dimension const& dimY = schema.getDimension("Y"); pdal::Dimension const& dimZ = schema.getDimension(z_name); boost::uint32_t numPoints = 0; double xd(0.0); double yd(0.0); double zd(0.0); for (boost::uint32_t pointIndex=0; pointIndex < data.getNumPoints(); pointIndex++) { boost::int32_t x = data.getField<boost::int32_t>(dimX, pointIndex); boost::int32_t y = data.getField<boost::int32_t>(dimY, pointIndex); boost::int32_t z = data.getField<boost::int32_t>(dimZ, pointIndex); xd = dimX.applyScaling<boost::int32_t>(x); yd = dimY.applyScaling<boost::int32_t>(y); zd = dimZ.applyScaling<boost::int32_t>(z); m_bounds.setMinimum(0, (std::min)(xd, m_bounds.getMinimum(0))); m_bounds.setMinimum(1, (std::min)(yd, m_bounds.getMinimum(1))); m_bounds.setMaximum(0, (std::max)(xd, m_bounds.getMaximum(0))); m_bounds.setMaximum(1, (std::max)(yd, m_bounds.getMaximum(1))); m_coordinates.push_back(boost::tuple<double, double, double>(xd, yd, zd)); numPoints++; } return numPoints; }
pdal::Bounds<double> Writer::CalculateBounds(PointBuffer const& buffer) { pdal::Schema const& schema = buffer.getSchema(); pdal::Bounds<double> output; boost::optional<Dimension const&> dimX = schema.getDimension("X"); boost::optional<Dimension const&> dimY = schema.getDimension("Y"); boost::optional<Dimension const&> dimZ = schema.getDimension("Z"); bool first = true; for (boost::uint32_t pointIndex=0; pointIndex<buffer.getNumPoints(); pointIndex++) { const boost::int32_t xi = buffer.getField<boost::int32_t>(*dimX, pointIndex); const boost::int32_t yi = buffer.getField<boost::int32_t>(*dimY, pointIndex); const boost::int32_t zi = buffer.getField<boost::int32_t>(*dimZ, pointIndex); const double xd = dimX->applyScaling(xi); const double yd = dimY->applyScaling(yi); const double zd = dimZ->applyScaling(zi); Vector<double> v(xd, yd, zd); if (first) { output = pdal::Bounds<double>(xd, yd, zd, xd, yd, zd); if (m_pcExtent.empty()) m_pcExtent = output; first = false; } output.grow(v); } m_pcExtent.grow(output); return output; }
bool Writer::WriteBlock(PointBuffer const& buffer) { bool bUsePartition = m_block_table_partition_column.size() != 0; // Pluck the block id out of the first point in the buffer pdal::Schema const& schema = buffer.getSchema(); Dimension const& blockDim = schema.getDimension("BlockID"); boost::int32_t block_id = buffer.getField<boost::int32_t>(blockDim, 0); std::ostringstream oss; std::ostringstream partition; if (bUsePartition) { partition << "," << m_block_table_partition_column; } oss << "INSERT INTO "<< m_block_table_name << "(OBJ_ID, BLK_ID, NUM_POINTS, POINTS, " "PCBLK_MIN_RES, BLK_EXTENT, PCBLK_MAX_RES, NUM_UNSORTED_POINTS, PT_SORT_DIM"; if (bUsePartition) oss << partition.str(); oss << ") " "VALUES ( :1, :2, :3, :4, 1, mdsys.sdo_geometry(:5, :6, null,:7, :8)" ", 1, 0, 1"; if (bUsePartition) oss << ", :9"; oss <<")"; // TODO: If gotdata == false below, this memory probably leaks --mloskot OCILobLocator** locator =(OCILobLocator**) VSIMalloc(sizeof(OCILobLocator*) * 1); Statement statement = Statement(m_connection->CreateStatement(oss.str().c_str())); long* p_pc_id = (long*) malloc(1 * sizeof(long)); p_pc_id[0] = m_pc_id; long* p_result_id = (long*) malloc(1 * sizeof(long)); p_result_id[0] = (long)block_id; long* p_num_points = (long*) malloc(1 * sizeof(long)); p_num_points[0] = (long)buffer.getNumPoints(); // std::cout << "point count on write: " << buffer.getNumPoints() << std::endl; // :1 statement->Bind(&m_pc_id); // :2 statement->Bind(p_result_id); // :3 statement->Bind(p_num_points); // :4 statement->Define(locator, 1); // std::vector<liblas::uint8_t> data; // bool gotdata = GetResultData(result, reader, data, 3); // if (! gotdata) throw std::runtime_error("unable to fetch point data byte array"); // boost::uint8_t* point_data = buffer.getData(0); boost::uint8_t* point_data; boost::uint32_t point_data_length; boost::uint32_t schema_byte_size; bool pack = getOptions().getValueOrDefault<bool>("pack_ignored_fields", true); if (pack) PackPointData(buffer, &point_data, point_data_length, schema_byte_size); else { point_data = buffer.getData(0); point_data_length = buffer.getSchema().getByteSize() * buffer.getNumPoints(); } // statement->Bind((char*)point_data,(long)(buffer.getSchema().getByteSize()*buffer.getNumPoints())); statement->Bind((char*)point_data,(long)(point_data_length)); // :5 long* p_gtype = (long*) malloc(1 * sizeof(long)); p_gtype[0] = m_gtype; statement->Bind(p_gtype); // :6 long* p_srid = 0; if (m_srid != 0) { p_srid = (long*) malloc(1 * sizeof(long)); p_srid[0] = m_srid; } statement->Bind(p_srid); // :7 OCIArray* sdo_elem_info=0; m_connection->CreateType(&sdo_elem_info, m_connection->GetElemInfoType()); SetElements(statement, sdo_elem_info); statement->Bind(&sdo_elem_info, m_connection->GetElemInfoType()); // :8 OCIArray* sdo_ordinates=0; m_connection->CreateType(&sdo_ordinates, m_connection->GetOrdinateType()); // x0, x1, y0, y1, z0, z1, bUse3d pdal::Bounds<double> bounds = CalculateBounds(buffer); SetOrdinates(statement, sdo_ordinates, bounds); statement->Bind(&sdo_ordinates, m_connection->GetOrdinateType()); // :9 long* p_partition_d = 0; if (bUsePartition) { p_partition_d = (long*) malloc(1 * sizeof(long)); p_partition_d[0] = m_block_table_partition_value; statement->Bind(p_partition_d); } try { statement->Execute(); } catch (std::runtime_error const& e) { std::ostringstream oss; oss << "Failed to insert block # into '" << m_block_table_name << "' table. Does the table exist? " << std::endl << e.what() << std::endl; throw std::runtime_error(oss.str()); } oss.str(""); OWStatement::Free(locator, 1); if (p_pc_id != 0) free(p_pc_id); if (p_result_id != 0) free(p_result_id); if (p_num_points != 0) free(p_num_points); if (p_gtype != 0) free(p_gtype); if (p_srid != 0) free(p_srid); if (p_partition_d != 0) free(p_partition_d); m_connection->DestroyType(&sdo_elem_info); m_connection->DestroyType(&sdo_ordinates); return true; }
bool Writer::WriteBlock(PointBuffer const& buffer) { boost::uint8_t* point_data; boost::uint32_t point_data_length; boost::uint32_t schema_byte_size; bool pack = getOptions().getValueOrDefault<bool>("pack_ignored_fields", true); if (pack) PackPointData(buffer, &point_data, point_data_length, schema_byte_size); else { point_data = buffer.getData(0); point_data_length = buffer.getSchema().getByteSize() * buffer.getNumPoints(); } bool doKDTree = getOptions().getValueOrDefault<bool>("kdtreeify", true); if (doKDTree) { // kdtreeify(buffer, &point_data, point_data_length, schema_byte_size); } std::string block_table = getOptions().getValueOrThrow<std::string>("block_table"); // // Pluck the block id out of the first point in the buffer pdal::Schema const& schema = buffer.getSchema(); Dimension const& blockDim = schema.getDimension("BlockID"); m_block_id = buffer.getField<boost::int32_t>(blockDim, 0); m_obj_id = getOptions().getValueOrThrow<boost::int32_t>("pc_id"); m_num_points = static_cast<boost::int64_t>(buffer.getNumPoints()); // if (m_type == DATABASE_POSTGRESQL) // { // std::string cloud_column = getOptions().getValueOrDefault<std::string>("cloud_column", "id"); // bool is3d = getOptions().getValueOrDefault<bool>("is3d", false); // // // std::vector<boost::uint8_t> block_data; // for (boost::uint32_t i = 0; i < point_data_length; ++i) // { // block_data.push_back(point_data[i]); // } // // if (pack) // delete point_data; // m_block_bytes.str(""); // Utils::binary_to_hex_stream(point_data, m_block_bytes, 0, point_data_length); // m_block_data = m_block_bytes.str(); // //std::cout << "hex: " << hex.substr(0, 30) << std::endl; // m_srid = getOptions().getValueOrDefault<boost::uint32_t>("srid", 4326); // // boost::uint32_t precision(9); // pdal::Bounds<double> bounds = buffer.calculateBounds(3); // // m_extent.str(""); // m_extent = bounds.toWKT(precision); // polygons are only 2d, not cubes // // m_bbox.str(""); // m_bbox = bounds.toBox(precision, 3); // log()->get(logDEBUG) << "extent: " << m_extent << std::endl; // log()->get(logDEBUG) << "bbox: " << m_bbox << std::endl; // // if (!m_block_statement) // { // // m_block_statement = (m_session->prepare << m_block_insert_query.str(), \ // // ::soci::use(m_obj_id, "obj_id"), \ // // ::soci::use(m_block_id, "block_id"), \ // // ::soci::use(m_num_points, "num_points"), \ // // ::soci::use(m_block_bytes.str(),"hex"), \ // // ::soci::use(m_extent.str(), "extent"), \ // // ::soci::use(m_srid, "srid"), \ // // ::soci::use(m_bbox.str(), "bbox")); // m_block_statement = new ::soci::statement(*m_session); // // m_block_statement->exchange(::soci::use(m_obj_id, "obj_id")); // m_block_statement->exchange(::soci::use(m_block_id, "block_id")); // m_block_statement->exchange(::soci::use(m_num_points, "num_points")); // m_block_statement->exchange(::soci::use(m_block_data,"hex")); // m_block_statement->exchange(::soci::use(m_extent, "extent")); // m_block_statement->exchange(::soci::use(m_srid, "srid")); // m_block_statement->exchange(::soci::use(m_bbox, "bbox")); // m_block_statement->alloc(); // m_block_statement->prepare(m_block_insert_query.str()); // m_block_statement->define_and_bind(); // // } // // ::soci::statement st = (m_session->prepare << m_block_insert_query.str(), \ // // ::soci::use(m_obj_id, "obj_id"), \ // // ::soci::use(m_block_id, "block_id"), \ // // ::soci::use(m_num_points, "num_points"), \ // // ::soci::use(m_block_bytes.str(),"hex"), \ // // ::soci::use(m_extent.str(), "extent"), \ // // ::soci::use(m_srid, "srid"), \ // // ::soci::use(m_bbox.str(), "bbox")); // try // { // m_block_statement->execute(true); // } // catch (std::exception const& e) // { // std::ostringstream oss; // oss << "Insert query failed with error '" << e.what() << "'"; // m_session->rollback(); // throw pdal_error(oss.str()); // } // // } return true; }
void Writer::PackPointData(PointBuffer const& buffer, boost::uint8_t** point_data, boost::uint32_t& point_data_len, boost::uint32_t& schema_byte_size) { // Creates a new buffer that has the ignored dimensions removed from // it. schema::index_by_index const& idx = buffer.getSchema().getDimensions().get<schema::index>(); schema_byte_size = 0; schema::index_by_index::size_type i(0); for (i = 0; i < idx.size(); ++i) { if (! idx[i].isIgnored()) schema_byte_size = schema_byte_size+idx[i].getByteSize(); } log()->get(logDEBUG) << "Packed schema byte size " << schema_byte_size; point_data_len = buffer.getNumPoints() * schema_byte_size; *point_data = new boost::uint8_t[point_data_len]; boost::uint8_t* current_position = *point_data; for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i) { boost::uint8_t* data = buffer.getData(i); for (boost::uint32_t d = 0; d < idx.size(); ++d) { if (! idx[d].isIgnored()) { memcpy(current_position, data, idx[d].getByteSize()); current_position = current_position+idx[d].getByteSize(); } data = data + idx[d].getByteSize(); } } // Create a vector of two element vectors that states how long of a byte // run to copy for the point based on whether or not the ignored/used // flag of the dimension switches. This is to a) eliminate panning through // the dimension multi_index repeatedly per point, and to b) eliminate // tons of small memcpy in exchange for larger ones. // std::vector< std::vector< boost::uint32_t> > runs; // // bool switched = idx[0].isIgnored(); // start at with the first dimension // for (boost::uint32_t d = 0; d < idx.size(); ++d) // { // std::vector<boost::uint32_t> t; // if ( idx[d].isIgnored()) // { // t.push_back(0); // t.push_back(idx[d].getByteSize()); // } // else // { // t.push_back(1); // t.push_back(idx[d].getByteSize()); // } // // runs.push_back(t); // } // // for (boost::uint32_t i = 0; i < buffer.getNumPoints(); ++i) // { // boost::uint8_t* data = buffer.getData(i); // for (std::vector< std::vector<boost::uint32_t> >::size_type d=0; d < runs.size(); d++) // // for (boost::uint32_t d = 0; d < idx.size(); ++d) // { // boost::uint32_t isUsed = runs[d][0]; // boost::uint32_t byteSize = runs[d][1]; // if (isUsed != 0) // { // memcpy(current_position, data, byteSize); // current_position = current_position+byteSize; // } // data = data + byteSize; // // } // } }
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer) { boost::uint32_t numPointsRead = 0; std::string const& query = getReader().getOptions().getValueOrThrow<std::string>("query"); ::soci::row block; ::soci::indicator ind = ::soci::i_null; ::soci::statement blocks = (m_session->prepare << query, ::soci::into(block, ind)); blocks.execute(); bool bDidRead = blocks.fetch(); // if (ind == ::soci::i_null) // { // // We have no points to return // getReader().log()->get(logDEBUG) << "Query returned no points" << std::endl; // return 0; // } // // size_t size = block.size(); // for (size_t i = 0; i < size; ++i) // { // getReader().log()->get(logDEBUG3) << "column: " << block.get_properties(i).get_name() << std::endl; // } if (!m_active_buffer) { m_active_buffer = fetchPointBuffer(block.get<int>("cloud_id"), block.get<std::string>("schema"), user_buffer.getCapacity()); m_active_cloud_id = block.get<int>("cloud_id"); } // // This shouldn't ever happen int num_points = block.get<int>("num_points"); if (num_points > static_cast<boost::int32_t>(m_active_buffer->getCapacity())) { std::ostringstream oss; oss << "Block size, " << num_points <<", is too large to fit in " << "buffer of size " << user_buffer.getCapacity() <<". Increase buffer capacity with writer's \"chunk_size\" option " << "or increase the read buffer size"; throw buffer_too_small(oss.str()); } while (bDidRead) { boost::uint32_t numReadThisBlock = static_cast<boost::uint32_t>(block.get<int>("num_points")); boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints(); getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: " << numReadThisBlock << " numSpaceLeftThisBlock: " << numSpaceLeftThisBuffer << " total numPointsRead: " << numPointsRead << std::endl; numPointsRead = numPointsRead + numReadThisBlock; readBlob(block, (std::min)(numReadThisBlock, numSpaceLeftThisBuffer)); fillUserBuffer(user_buffer); bDidRead = blocks.fetch(); // if (!bDidRead) // return user_buffer.getNumPoints(); boost::int32_t const& current_cloud_id = block.get<int>("cloud_id"); if (current_cloud_id != m_active_cloud_id) { getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: " << current_cloud_id << " m_active_cloud_id: " << m_active_cloud_id << std::endl; m_active_buffer = fetchPointBuffer(current_cloud_id, block.get<std::string>("schema"), user_buffer.getCapacity()); m_active_cloud_id = current_cloud_id; return user_buffer.getNumPoints(); } } return numPointsRead; }
void Diff::checkPoints( StageSequentialIterator* source_iter, PointBuffer& source_data, StageSequentialIterator* candidate_iter, PointBuffer& candidate_data, ptree& errors) { boost::uint32_t i(0); boost::uint32_t chunk(0); boost::uint32_t MAX_BADBYTES(20); boost::uint32_t badbytes(0); while (!source_iter->atEnd()) { const boost::uint32_t numSrcRead = source_iter->read(source_data); const boost::uint32_t numCandidateRead = candidate_iter->read(candidate_data); if (numSrcRead != numCandidateRead) { std::ostringstream oss; oss << "Unable to read same number of points for chunk number"; errors.put<std::string>("points.error", oss.str()); errors.put<boost::uint32_t>("points.candidate" , numCandidateRead); errors.put<boost::uint32_t>("points.source" , numSrcRead); } chunk++; pdal::pointbuffer::PointBufferByteSize source_byte_length(0); pdal::pointbuffer::PointBufferByteSize candidate_byte_length(0); source_byte_length = static_cast<pdal::pointbuffer::PointBufferByteSize>(source_data.getSchema().getByteSize()) * static_cast<pdal::pointbuffer::PointBufferByteSize>(source_data.getNumPoints()); candidate_byte_length = static_cast<pdal::pointbuffer::PointBufferByteSize>(candidate_data.getSchema().getByteSize()) * static_cast<pdal::pointbuffer::PointBufferByteSize>(candidate_data.getNumPoints()); if (source_byte_length != candidate_byte_length) { std::ostringstream oss; oss << "Source byte length != candidate byte length"; errors.put<std::string>("buffer.error", oss.str()); errors.put<boost::uint32_t>("buffer.candidate" , candidate_byte_length); errors.put<boost::uint32_t>("buffer.source" , source_byte_length); } boost::uint8_t* s = source_data.getData(0); boost::uint8_t* c = candidate_data.getData(0); for (boost::uint32_t p = 0; p < std::min(source_byte_length, candidate_byte_length); ++p) { if (s[p] != c[p]) { std::ostringstream oss; oss << "Byte number " << p << " is not equal for source and candidate"; errors.put<std::string>("data.error", oss.str()); badbytes++; } } if (badbytes > MAX_BADBYTES ) break; } }
// append all points from src buffer to end of dst buffer, based on the our bounds boost::uint32_t Crop::processBuffer(PointBuffer& dstData, const PointBuffer& srcData) const { const Schema& schema = dstData.getSchema(); const Bounds<double>& bounds = this->getBounds(); boost::uint32_t numSrcPoints = srcData.getNumPoints(); boost::uint32_t dstIndex = dstData.getNumPoints(); boost::uint32_t numPointsAdded = 0; boost::optional<Dimension const&> dimX = schema.getDimension("X"); boost::optional<Dimension const&> dimY = schema.getDimension("Y"); boost::optional<Dimension const&> dimZ = schema.getDimension("Z"); for (boost::uint32_t srcIndex=0; srcIndex<numSrcPoints; srcIndex++) { // need to scale the values double x(0.0); double y(0.0); double z(0.0); if (dimX->getInterpretation() == dimension::SignedInteger ) { boost::int32_t xi = srcData.getField<boost::int32_t>(*dimX, srcIndex); boost::int32_t yi = srcData.getField<boost::int32_t>(*dimY, srcIndex); boost::int32_t zi = srcData.getField<boost::int32_t>(*dimZ, srcIndex); x = dimX->applyScaling(xi); y = dimY->applyScaling(yi); z = dimZ->applyScaling(zi); } else if (dimX->getInterpretation() == dimension::UnsignedInteger) { boost::uint32_t xi = srcData.getField<boost::uint32_t>(*dimX, srcIndex); boost::uint32_t yi = srcData.getField<boost::uint32_t>(*dimY, srcIndex); boost::uint32_t zi = srcData.getField<boost::uint32_t>(*dimZ, srcIndex); x = dimX->applyScaling(xi); y = dimY->applyScaling(yi); z = dimZ->applyScaling(zi); } else { x = srcData.getField<double>(*dimX, srcIndex); y = srcData.getField<double>(*dimY, srcIndex); z = srcData.getField<double>(*dimZ, srcIndex); } Vector<double> point(x,y,z); if (bounds.contains(point)) { dstData.copyPointFast(dstIndex, srcIndex, srcData); dstData.setNumPoints(dstIndex+1); ++dstIndex; ++numPointsAdded; } } assert(dstIndex <= dstData.getCapacity()); return numPointsAdded; }
void IteratorBase::fillUserBuffer(PointBuffer& user_buffer) { Schema const& user_schema = user_buffer.getSchema(); schema::index_by_index const& idx = user_schema.getDimensions().get<schema::index>(); boost::int32_t numOraclePoints = m_oracle_buffer->getNumPoints() - m_buffer_position; boost::int32_t numUserSpace = user_buffer.getCapacity() - user_buffer.getNumPoints(); boost::int32_t howManyThisRead = (std::min)(numUserSpace, numOraclePoints); if (numUserSpace < 0) { std::ostringstream oss; oss << "numUserSpace < 0! : " << numUserSpace; throw pdal_error(oss.str()); } if (numOraclePoints < 0) { std::ostringstream oss; oss << "numOraclePoints < 0! : " << numOraclePoints; throw pdal_error(oss.str()); } if (user_buffer.getNumPoints() + howManyThisRead > user_buffer.getCapacity()) { std::ostringstream oss; oss << "user_buffer.getNumPoints() + howManyThisRead == " << user_buffer.getNumPoints() + howManyThisRead << " but user_buffer.getCapacity() == " << user_buffer.getCapacity(); throw pdal_error(oss.str()); } PointBuffer::copyLikeDimensions(*m_oracle_buffer, user_buffer, *m_dimension_map, m_buffer_position, user_buffer.getNumPoints(), howManyThisRead); Schema const& src = m_oracle_buffer->getSchema(); Dimension const& src_x = src.getDimension("drivers.oci.reader.X"); Dimension const& src_y = src.getDimension("drivers.oci.reader.Y"); Dimension const& src_z = src.getDimension("drivers.oci.reader.Z"); Schema const& dst = user_buffer.getSchema(); Dimension const& dst_x = dst.getDimension("drivers.oci.reader.X"); Dimension const& dst_y = dst.getDimension("drivers.oci.reader.Y"); Dimension const& dst_z = dst.getDimension("drivers.oci.reader.Z"); bool bDifferentScales = (!pdal::Utils::compare_distance(dst_x.getNumericScale(), src_x.getNumericScale()) || !pdal::Utils::compare_distance(dst_y.getNumericScale(), src_y.getNumericScale()) || !pdal::Utils::compare_distance(dst_z.getNumericScale(), src_z.getNumericScale())); bool bDifferentOffsets = (!pdal::Utils::compare_distance(dst_x.getNumericOffset(), src_x.getNumericOffset()) || !pdal::Utils::compare_distance(dst_y.getNumericOffset(), src_y.getNumericOffset()) || !pdal::Utils::compare_distance(dst_z.getNumericOffset(), src_z.getNumericOffset())); bool bNormalizeXYZ = getReader().getOptions().getValueOrDefault<bool>("do_normalize_xyz", true); if ((bDifferentScales || bDifferentOffsets) && bNormalizeXYZ) { for (unsigned i = m_buffer_position; i < howManyThisRead; ++i) { double x = m_oracle_buffer->applyScaling(src_x, i); boost::int32_t xi = dst_x.removeScaling<boost::int32_t>(x); user_buffer.setField<boost::int32_t>(dst_x, user_buffer.getNumPoints()+i, xi); double y = m_oracle_buffer->applyScaling(src_y, i); boost::int32_t yi = dst_y.removeScaling<boost::int32_t>(y); user_buffer.setField<boost::int32_t>(dst_y, user_buffer.getNumPoints()+i, yi); double z = m_oracle_buffer->applyScaling(src_z, i); boost::int32_t zi = dst_z.removeScaling<boost::int32_t>(z); user_buffer.setField<boost::int32_t>(dst_z, user_buffer.getNumPoints()+i, zi); } } getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: " << m_buffer_position << std::endl; if (numOraclePoints > numUserSpace) m_buffer_position = m_buffer_position + numUserSpace; else if (numOraclePoints < numUserSpace) m_buffer_position = 0; getReader().log()->get(logDEBUG2) << "IteratorBase::fillUserBuffer m_buffer_position: " << m_buffer_position << std::endl; bool bSetPointSourceId = getReader().getOptions().getValueOrDefault<bool>("populate_pointsourceid", false); if (bSetPointSourceId) { Dimension const* point_source_field = user_buffer.getSchema().getDimensionPtr("PointSourceId"); if (point_source_field) { for (boost::int32_t i = m_buffer_position; i < howManyThisRead; ++i) { assert(user_buffer.getNumPoints() + i < user_buffer.getCapacity()); if (point_source_field->getByteSize() == 2) user_buffer.setField(*point_source_field, user_buffer.getNumPoints() + i, static_cast<boost::uint16_t>(m_active_cloud_id)); else if (point_source_field->getByteSize() == 4) { user_buffer.setField(*point_source_field, user_buffer.getNumPoints() + i, m_active_cloud_id); } } } } user_buffer.setNumPoints(howManyThisRead + user_buffer.getNumPoints()); }
void Delta::outputDetail(PointBuffer& source_data, IndexedPointBuffer& candidate_data, std::map<Point, Point> *points) const { Schema const& candidate_schema = candidate_data.getSchema(); Dimension const& cDimX = candidate_schema.getDimension("X"); Dimension const& cDimY = candidate_schema.getDimension("Y"); Dimension const& cDimZ = candidate_schema.getDimension("Z"); Schema const& source_schema = source_data.getSchema(); Dimension const& sDimX = source_schema.getDimension("X"); Dimension const& sDimY = source_schema.getDimension("Y"); Dimension const& sDimZ = source_schema.getDimension("Z"); bool bWroteHeader(false); std::ostream& ostr = m_outputStream ? *m_outputStream : std::cout; candidate_data.build(m_3d); boost::uint32_t count(std::min(source_data.getNumPoints(), candidate_data.getNumPoints())); for (boost::uint32_t i = 0; i < count; ++i) { double sx = source_data.applyScaling(sDimX, i); double sy = source_data.applyScaling(sDimY, i); double sz = source_data.applyScaling(sDimZ, i); std::vector<std::size_t> ids = candidate_data.neighbors(sx, sy, sz, 1); if (!ids.size()) { std::ostringstream oss; oss << "unable to find point for id '" << i <<"'"; throw app_runtime_error(oss.str() ); } std::size_t id = ids[0]; double cx = candidate_data.applyScaling(cDimX, id); double cy = candidate_data.applyScaling(cDimY, id); double cz = candidate_data.applyScaling(cDimZ, id); Point s(sx, sy, sz, id); Point c(cx, cy, cz, id); double xd = sx - cx; double yd = sy - cy; double zd = sz - cz; if (!bWroteHeader) { writeHeader(ostr, m_3d); bWroteHeader = true; } ostr << i << ","; boost::uint32_t precision = Utils::getStreamPrecision(cDimX.getNumericScale()); ostr.setf(std::ios_base::fixed, std::ios_base::floatfield); ostr.precision(precision); ostr << xd << ","; precision = Utils::getStreamPrecision(cDimY.getNumericScale()); ostr.precision(precision); ostr << yd; if (m_3d) { ostr << ","; precision = Utils::getStreamPrecision(cDimZ.getNumericScale()); ostr.precision(precision); ostr << zd; } ostr << std::endl; } if (m_outputStream) { FileUtils::closeFile(m_outputStream); } }
boost::uint32_t IteratorBase::myReadBlocks(PointBuffer& user_buffer) { boost::uint32_t numPointsRead = 0; user_buffer.setNumPoints(0); bool bDidRead = false; if (!m_oracle_buffer) { m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc); if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!"); m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer); boost::int32_t current_cloud_id(0); current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id); m_active_cloud_id = current_cloud_id; } // This shouldn't ever happen if (m_block->num_points > static_cast<boost::int32_t>(m_oracle_buffer->getCapacity())) { m_oracle_buffer->resize(m_block->num_points); } if (!m_block->num_points) { // We still have a block of data from the last readBuffer call // that was partially read. getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: fetching first block" << std::endl; bDidRead = m_initialQueryStatement->Fetch(); if (!bDidRead) { m_at_end = true; return 0; } user_buffer.setSpatialBounds(getBounds(m_initialQueryStatement, m_block)); } else { // Our read was already "done" last readBuffer call, but if we're done, // we're done if (m_at_end) getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we are at end of the blocks;" << std::endl; else getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: we have points left to read on this block" << std::endl; if (m_at_end) return 0; bDidRead = true; } while (bDidRead) { boost::uint32_t numReadThisBlock = m_block->num_points; boost::uint32_t numSpaceLeftThisBuffer = user_buffer.getCapacity() - user_buffer.getNumPoints(); getReader().log()->get(logDEBUG4) << "IteratorBase::myReadBlocks:" "numReadThisBlock: " << numReadThisBlock << " numSpaceLeftThisBlock: " << numSpaceLeftThisBuffer << " total numPointsRead: " << numPointsRead << std::endl; numPointsRead = numPointsRead + numReadThisBlock; readBlob(m_initialQueryStatement, m_block, m_block->num_points); fillUserBuffer(user_buffer); if (m_buffer_position != 0) { return user_buffer.getNumPoints(); } else { bDidRead = m_initialQueryStatement->Fetch(); if (!bDidRead) { getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: done reading block. Read " << numPointsRead << " points" << std::endl; m_at_end = true; return user_buffer.getNumPoints(); } } boost::int32_t current_cloud_id(0); current_cloud_id = m_initialQueryStatement->GetInteger(&m_block->pc->pc_id); getReader().log()->get(logDEBUG3) << "IteratorBase::myReadBlocks: current_cloud_id: " << current_cloud_id << " m_active_cloud_id: " << m_active_cloud_id << std::endl; if (current_cloud_id != m_active_cloud_id) { m_oracle_buffer = fetchPointBuffer(m_initialQueryStatement, m_block->pc); if (!m_oracle_buffer) throw pdal_error("m_oracle_buffer was NULL!"); m_dimension_map = fetchDimensionMap(m_initialQueryStatement, m_block->pc, *m_oracle_buffer, user_buffer); m_active_cloud_id = current_cloud_id; return user_buffer.getNumPoints(); } } return numPointsRead; }