/// Write a point's packed data into a buffer. /// \param[in] view PointView to write to. /// \param[in] idx Index of point to write. /// \param[in] buf Pointer to packed DB point data. void DbReader::writePoint(PointView& view, PointId idx, const char *buf) { using namespace Dimension; for (auto di = m_dims.begin(); di != m_dims.end(); ++di) { DimType dimType = di->m_dimType; // If we read X, Y or Z as a signed 32, apply the transform and write // the transformed value (double). if (dimType.m_type == Type::Signed32 && (dimType.m_id == Id::X || dimType.m_id == Id::Y || dimType.m_id == Id::Z)) { int32_t i; memcpy(&i, buf, sizeof(int32_t)); double d = (i * dimType.m_xform.m_scale) + dimType.m_xform.m_offset; view.setField(dimType.m_id, idx, d); } else { view.setField(dimType.m_id, dimType.m_type, idx, buf); } buf += Dimension::size(dimType.m_type); } }
void ColorizationFilter::filter(PointView& view) { int32_t pixel(0); int32_t line(0); std::array<double, 2> pix = { {0.0, 0.0} }; for (PointId idx = 0; idx < view.size(); ++idx) { double x = view.getFieldAs<double>(Dimension::Id::X, idx); double y = view.getFieldAs<double>(Dimension::Id::Y, idx); if (!getPixelAndLinePosition(x, y, m_inverse_transform, pixel, line, m_ds)) continue; for (auto bi = m_bands.begin(); bi != m_bands.end(); ++bi) { BandInfo& b = *bi; GDALRasterBandH hBand = GDALGetRasterBand(m_ds, b.m_band); if (hBand == NULL) { std::ostringstream oss; oss << "Unable to get band " << b.m_band << " from data source!"; throw pdal_error(oss.str()); } if (GDALRasterIO(hBand, GF_Read, pixel, line, 1, 1, &pix[0], 1, 1, GDT_CFloat64, 0, 0) == CE_None) view.setField(b.m_dim, idx, pix[0] * b.m_scale); } } }
void AttributeFilter::filter(PointView& view) { if (m_value == m_value) for (PointId i = 0; i < view.size(); ++i) view.setField(m_dim, i, m_value); else UpdateGEOSBuffer(view); }
void ColorizationFilter::filter(PointView& view) { PointRef point = view.point(0); for (PointId idx = 0; idx < view.size(); ++idx) { point.setPointId(idx); processOne(point); } }
void HexBin::filter(PointView& view) { for (PointId idx = 0; idx < view.size(); ++idx) { double x = view.getFieldAs<double>(pdal::Dimension::Id::X, idx); double y = view.getFieldAs<double>(pdal::Dimension::Id::Y, idx); m_grid->addPoint(x, y); } m_count += view.size(); }
PDAL_DLL Eigen::MatrixXd pointViewToEigen(const PointView& view) { Eigen::MatrixXd matrix(view.size(), 3); for (PointId i = 0; i < view.size(); ++i) { matrix(i, 0) = view.getFieldAs<double>(Dimension::Id::X, i); matrix(i, 1) = view.getFieldAs<double>(Dimension::Id::Y, i); matrix(i, 2) = view.getFieldAs<double>(Dimension::Id::Z, i); } return matrix; }
void EstimateRankFilter::filter(PointView& view) { KD3Index& kdi = view.build3dIndex(); for (PointId i = 0; i < view.size(); ++i) { // find the k-nearest neighbors auto ids = kdi.neighbors(i, m_knn); view.setField(m_rank, i, eigen::computeRank(view, ids, m_thresh)); } }
void ReprojectionFilter::filter(PointView& view) { for (PointId id = 0; id < view.size(); ++id) { double x = view.getFieldAs<double>(Dimension::Id::X, id); double y = view.getFieldAs<double>(Dimension::Id::Y, id); double z = view.getFieldAs<double>(Dimension::Id::Z, id); transform(x, y, z); view.setField(Dimension::Id::X, id, x); view.setField(Dimension::Id::Y, id, y); view.setField(Dimension::Id::Z, id, z); } }
void HAGFilter::filter(PointView& view) { PointViewPtr gView = view.makeNew(); PointViewPtr ngView = view.makeNew(); std::vector<PointId> gIdx, ngIdx; // First pass: Separate into ground and non-ground views. for (PointId i = 0; i < view.size(); ++i) { double c = view.getFieldAs<double>(Dimension::Id::Classification, i); if (c == 2) { gView->appendPoint(view, i); gIdx.push_back(i); } else { ngView->appendPoint(view, i); ngIdx.push_back(i); } } // Bail if there weren't any points classified as ground. if (gView->size() == 0) throwError("Input PointView does not have any points classified " "as ground"); // Build the 2D KD-tree. KD2Index kdi(*gView); kdi.build(); // Second pass: Find Z difference between non-ground points and the nearest // neighbor (2D) in the ground view. for (PointId i = 0; i < ngView->size(); ++i) { PointRef point = ngView->point(i); double z0 = point.getFieldAs<double>(Dimension::Id::Z); auto ids = kdi.neighbors(point, 1); double z1 = gView->getFieldAs<double>(Dimension::Id::Z, ids[0]); view.setField(Dimension::Id::HeightAboveGround, ngIdx[i], z0 - z1); } // Final pass: Ensure that all ground points have height value pegged at 0. for (auto const& i : gIdx) view.setField(Dimension::Id::HeightAboveGround, i, 0.0); }
void DbReader::writeField(PointView& view, const char *pos, const DimType& dim, PointId idx) { using namespace Dimension; if (dim.m_id == Id::X || dim.m_id == Id::Y || dim.m_id == Id::Z) { Everything e; memcpy(&e, pos, Dimension::size(dim.m_type)); double d = Utils::toDouble(e, dim.m_type); d = (d * dim.m_xform.m_scale.m_val) + dim.m_xform.m_offset.m_val; view.setField(dim.m_id, idx, d); } else view.setField(dim.m_id, dim.m_type, idx, pos); }
void TransformationFilter::filter(PointView& view) { PointRef point(view, 0); for (PointId idx = 0; idx < view.size(); ++idx) { point.setPointId(idx); processOne(point); } }
void FerryFilter::filter(PointView& view) { PointRef point(view, 0); for (PointId id = 0; id < view.size(); ++id) { point.setPointId(id); processOne(point); } }
void ProgrammableFilter::filter(PointView& view) { log()->get(LogLevel::Debug5) << "Python script " << *m_script << " processing " << view.size() << " points." << std::endl; m_pythonMethod->resetArguments(); m_pythonMethod->begin(view, m_totalMetadata); m_pythonMethod->execute(); m_pythonMethod->end(view, getMetadata()); }
void AttributeFilter::filter(PointView& view) { for (auto& dim_par : m_dimensions) { if (dim_par.second.isogr) { UpdateGEOSBuffer(view, dim_par.second); } else { for (PointId i = 0; i < view.size(); ++i) { double v = boost::lexical_cast<double>(dim_par.second.value); view.setField(dim_par.second.dim, i, v); } } } }
void DbReader::writeField(PointView& view, const char *pos, const DimType& dim, PointId idx) { using namespace Dimension; if (dim.m_type == Type::Signed32 && (dim.m_id == Id::X || dim.m_id == Id::Y || dim.m_id == Id::Z)) { int32_t i; memcpy(&i, pos, sizeof(int32_t)); double d = (i * dim.m_xform.m_scale) + dim.m_xform.m_offset; view.setField(dim.m_id, idx, d); } else { view.setField(dim.m_id, dim.m_type, idx, pos); } }
void ColorizationFilter::filter(PointView& view) { std::vector<double> data; for (PointId idx = 0; idx < view.size(); ++idx) { double x = view.getFieldAs<double>(Dimension::Id::X, idx); double y = view.getFieldAs<double>(Dimension::Id::Y, idx); if (!m_raster->read(x, y, data)) continue; int i(0); for (auto bi = m_bands.begin(); bi != m_bands.end(); ++bi) { BandInfo& b = *bi; view.setField(b.m_dim, idx, data[i] * b.m_scale); ++i; } } }
PointRef& operator=(const PointRef& r) { assert(m_buf == NULL || r.m_buf == m_buf); if (!m_buf) { m_buf = r.m_buf; m_id = m_buf->getTemp(r.m_id); m_tmp = true; } else m_buf->m_index[m_id] = r.m_buf->m_index[r.m_id]; return *this; }
/// Read a field from a PointView and write its value as formatted for output /// to the DB schema to the location as requested. /// \param[in] view PointView to read from. /// \param[in] pos Location in which to store field value. /// \param[in] id ID of the dimension to read. /// \param[in] idx Index of point to read. /// \return Size of field as read. size_t DbWriter::readField(const PointView& view, char *pos, Dimension::Id::Enum id, PointId idx) { using namespace Dimension; DimType& dt = m_dimMap[(int)id]; size_t size = Dimension::size(dt.m_type); // Using the ID instead of a dimType as the arugment hides the complication // of the "type" of the dimension to retrieve in the case of location // scaling. view.getField(pos, id, dt.m_type, idx); auto iconvert = [pos](const XForm& xform, Dimension::Id::Enum dim) { double d; int32_t i; memcpy(&d, pos, sizeof(double)); d = (d - xform.m_offset) / xform.m_scale; if (!Utils::numericCast(d, i)) { std::ostringstream oss; oss << "Unable to convert double to int32 for packed DB output: "; oss << Dimension::name(dim) << ": (" << d << ")."; throw pdal_error(oss.str()); } memcpy(pos, &i, sizeof(int32_t)); }; if (m_locationScaling) { // For X, Y or Z. if (id == Id::X) { iconvert(m_xXform, Id::X); size = sizeof(int32_t); } else if (id == Id::Y) { iconvert(m_yXform, Id::Y); size = sizeof(int32_t); } else if (id == Id::Z) { iconvert(m_zXform, Id::Z); size = sizeof(int32_t); } } return size; }
void BufferedInvocation::end(PointView& view, MetadataNode m) { // for each entry in the script's outs dictionary, // look up that entry's name in the schema and then // copy the data into the right dimension spot in the // buffer std::vector<std::string> names; getOutputNames(names); PointLayoutPtr layout(view.m_pointTable.layout()); Dimension::IdList const& dims = layout->dims(); for (auto di = dims.begin(); di != dims.end(); ++di) { Dimension::Id::Enum d = *di; const Dimension::Detail *dd = layout->dimDetail(d); std::string name = layout->dimName(*di); auto found = std::find(names.begin(), names.end(), name); if (found == names.end()) continue; // didn't have this dim in the names assert(name == *found); assert(hasOutputVariable(name)); size_t size = dd->size(); void *data = extractResult(name, dd->type()); char *p = (char *)data; for (PointId idx = 0; idx < view.size(); ++idx) { view.setField(d, dd->type(), idx, (void *)p); p += size; } } for (auto bi = m_buffers.begin(); bi != m_buffers.end(); ++bi) free(*bi); m_buffers.clear(); addMetadata(m_metaOut, m); }
void DiffKernel::checkPoints(const PointView& source_data, const PointView& candidate_data, MetadataNode errors) { uint32_t MAX_BADBYTES(20); uint32_t badbytes(0); // Both schemas have already been determined to be equal, so are the // same size and in the same order. Dimension::IdList const& sourceDims = source_data.dims(); Dimension::IdList const& candidateDims = candidate_data.dims(); char sbuf[8]; char cbuf[8]; for (PointId idx = 0; idx < source_data.size(); ++idx) { for (size_t d = 0; d < sourceDims.size(); ++d) { Dimension::Id sd = sourceDims[d]; Dimension::Id cd = candidateDims[d]; source_data.getRawField(sd, idx, (void *)sbuf); candidate_data.getRawField(cd, idx, (void *)cbuf); Dimension::Type t = Dimension::defaultType(cd); size_t size = Dimension::size(t); if (memcmp(sbuf, cbuf, size)) { std::ostringstream oss; oss << "Point " << idx << " differs for dimension \"" << Dimension::name(sd) << "\" for source and candidate"; errors.add("data.error", oss.str()); badbytes++; } } if (badbytes > MAX_BADBYTES ) break; } }
void BufferedInvocation::begin(PointView& view, MetadataNode m) { PointLayoutPtr layout(view.m_pointTable.layout()); Dimension::IdList const& dims = layout->dims(); for (auto di = dims.begin(); di != dims.end(); ++di) { Dimension::Id::Enum d = *di; const Dimension::Detail *dd = layout->dimDetail(d); void *data = malloc(dd->size() * view.size()); m_buffers.push_back(data); // Hold pointer for deallocation char *p = (char *)data; for (PointId idx = 0; idx < view.size(); ++idx) { view.getFieldInternal(d, idx, (void *)p); p += dd->size(); } std::string name = layout->dimName(*di); insertArgument(name, (uint8_t *)data, dd->type(), view.size()); } Py_XDECREF(m_metaIn); m_metaIn = plang::fromMetadata(m); }
Eigen::MatrixXd extendedLocalMinimum(PointView& view, int rows, int cols, double cell_size, BOX2D bounds) { using namespace Dimension; using namespace Eigen; // Index elevation values by row and column. std::map<uint32_t, std::vector<double>> hash; for (PointId i = 0; i < view.size(); ++i) { double x = view.getFieldAs<double>(Id::X, i); double y = view.getFieldAs<double>(Id::Y, i); double z = view.getFieldAs<double>(Id::Z, i); int c = Utils::clamp(static_cast<int>(floor(x-bounds.minx)/cell_size), 0, cols-1); int r = Utils::clamp(static_cast<int>(floor(y-bounds.miny)/cell_size), 0, rows-1); hash[r*cols+c].push_back(z); } // For each grid cell, sort elevations and detect local minimum, rejecting low // outliers. MatrixXd ZImin(rows, cols); ZImin.setConstant(std::numeric_limits<double>::quiet_NaN()); for (int c = 0; c < cols; ++c) { for (int r = 0; r < rows; ++r) { std::vector<double> cp(hash[r*cols+c]); if (cp.empty()) continue; std::sort(cp.begin(), cp.end()); if (cp.size() == 1) { ZImin(r, c) = cp[0]; continue; } for (size_t i = 0; i < cp.size()-1; ++i) { if (std::fabs(cp[i] - cp[i+1]) < 1.0) { ZImin(r, c) = cp[i]; break; } } } } return ZImin; }
void KDistanceFilter::filter(PointView& view) { using namespace Dimension; // Build the 3D KD-tree. log()->get(LogLevel::Debug) << "Building 3D KD-tree...\n"; KD3Index index(view); index.build(); // Increment the minimum number of points, as knnSearch will be returning // the neighbors along with the query point. m_k++; // Compute the k-distance for each point. The k-distance is the Euclidean // distance to k-th nearest neighbor. log()->get(LogLevel::Debug) << "Computing k-distances...\n"; for (PointId i = 0; i < view.size(); ++i) { std::vector<PointId> indices(m_k); std::vector<double> sqr_dists(m_k); index.knnSearch(i, m_k, &indices, &sqr_dists); view.setField(m_kdist, i, std::sqrt(sqr_dists[m_k-1])); } }
point_count_t OciReader::readDimMajor(PointView& view, BlockPtr block, point_count_t numPts) { using namespace Dimension; point_count_t numRemaining = block->numRemaining(); PointId startId = view.size(); point_count_t blockRemaining = numRemaining; point_count_t numRead = 0; DimTypeList dims = dbDimTypes(); for (auto di = dims.begin(); di != dims.end(); ++di) { PointId nextId = startId; char *pos = seekDimMajor(*di, block); blockRemaining = numRemaining; numRead = 0; while (numRead < numPts && blockRemaining > 0) { writeField(view, pos, *di, nextId); pos += Dimension::size(di->m_type); if (di->m_id == Id::PointSourceId && m_updatePointSourceId) view.setField(Id::PointSourceId, nextId, block->obj_id); if (m_cb && di == dims.rbegin().base() - 1) m_cb(view, nextId); nextId++; numRead++; blockRemaining--; } } block->setNumRemaining(blockRemaining); return numRead; }
/// Read a point's data packed into a buffer. /// \param[in] view PointView to read from. /// \param[in] idx Index of point to read. /// \param[in] outbuf Buffer to write to. /// \return Number of bytes written to buffer. size_t DbWriter::readPoint(const PointView& view, PointId idx, char *outbuf) { view.getPackedPoint(m_dimTypes, idx, outbuf); auto iconvert = [](const XForm& xform, const char *inpos, char *outpos) { double d; int32_t i; memcpy(&d, inpos, sizeof(double)); d = (d - xform.m_offset) / xform.m_scale; i = boost::numeric_cast<int32_t>(lround(d)); memcpy(outpos, &i, sizeof(int32_t)); }; if (m_locationScaling) { int outOffset; if (m_xPackedOffset >= 0) outOffset = m_xPackedOffset; else if (m_yPackedOffset >= 0) outOffset = m_yPackedOffset; else if (m_zPackedOffset >= 0) outOffset = m_zPackedOffset; else outOffset = m_packedPointSize; //So we return the proper size. if (m_xPackedOffset >= 0) { iconvert(m_xXform, outbuf + m_xPackedOffset, outbuf + outOffset); outOffset += sizeof(int); } if (m_yPackedOffset >= 0) { iconvert(m_yXform, outbuf + m_yPackedOffset, outbuf + outOffset); outOffset += sizeof(int); } if (m_zPackedOffset >= 0) { iconvert(m_zXform, outbuf + m_zPackedOffset, outbuf + outOffset); outOffset += sizeof(int); } return outOffset; } else return m_packedPointSize; }
point_count_t OciReader::readPointMajor(PointView& view, BlockPtr block, point_count_t numPts) { size_t numRemaining = block->numRemaining(); PointId nextId = view.size(); point_count_t numRead = 0; if (m_compression) { #ifdef PDAL_HAVE_LAZPERF LazPerfBuf buf(block->chunk); LazPerfDecompressor<LazPerfBuf> decompressor(buf, dbDimTypes()); std::vector<char> ptBuf(decompressor.pointSize()); while (numRead < numPts && numRemaining > 0) { point_count_t numWritten = decompressor.decompress(ptBuf.data(), ptBuf.size()); writePoint(view, nextId, ptBuf.data()); if (m_cb) m_cb(view, nextId); numRemaining--; nextId++; numRead++; } #else throw pdal_error("Can't decompress without LAZperf."); #endif } else { char *pos = seekPointMajor(block); while (numRead < numPts && numRemaining > 0) { writePoint(view, nextId, pos); if (m_cb) m_cb(view, nextId); pos += packedPointSize(); numRemaining--; nextId++; numRead++; } } block->setNumRemaining(numRemaining); return numRead; }
point_count_t LasWriter::fillWriteBuf(const PointView& view, PointId startId, std::vector<char>& buf) { point_count_t blocksize = buf.size() / m_lasHeader.pointLen(); blocksize = (std::min)(blocksize, view.size() - startId); PointId lastId = startId + blocksize; LeInserter ostream(buf.data(), buf.size()); PointRef point = (const_cast<PointView&>(view)).point(0); for (PointId idx = startId; idx < lastId; idx++) { point.setPointId(idx); fillPointBuf(point, ostream); } return blocksize; }
point_count_t OciReader::readPointMajor(PointView& view, BlockPtr block, point_count_t numPts) { PointId nextId = view.size(); point_count_t numRead = 0; if (m_compression) { #ifdef PDAL_HAVE_LAZPERF auto cb = [this, &view, &nextId, &numRead](char *buf, size_t bufsize) { writePoint(view, nextId, buf); if (m_cb) m_cb(view, nextId); nextId++; numRead++; }; const char *src = reinterpret_cast<const char *>(block->chunk.data()); size_t srcsize = block->chunk.size(); LazPerfDecompressor(cb, dbDimTypes(), block->numRemaining()). decompress(src, srcsize); #else throwError("Can't decompress without LAZperf."); #endif } else { char *pos = seekPointMajor(block); size_t cnt = block->numRemaining(); while (numRead < numPts && cnt--) { writePoint(view, nextId, pos); if (m_cb) m_cb(view, nextId); pos += packedPointSize(); nextId++; numRead++; } } block->setNumRemaining(block->numRemaining() - numRead); return numRead; }
size_t DbWriter::readField(const PointView& view, char *pos, DimType dimType, PointId idx) { using namespace Dimension; size_t size = Dimension::size(dimType.m_type); view.getField(pos, dimType.m_id, dimType.m_type, idx); auto iconvert = [pos](const XForm& xform) { double d; int32_t i; memcpy(&d, pos, sizeof(double)); d = (d - xform.m_offset) / xform.m_scale; i = boost::numeric_cast<int32_t>(lround(d)); memcpy(pos, &i, sizeof(int32_t)); }; if (m_locationScaling) { // For X, Y or Z. if (dimType.m_id == Id::X) { iconvert(m_xXform); size = sizeof(int32_t); } else if (dimType.m_id == Id::Y) { iconvert(m_yXform); size = sizeof(int32_t); } else if (dimType.m_id == Id::Z) { iconvert(m_zXform); size = sizeof(int32_t); } } return size; }
void NormalFilter::filter(PointView& view) { KD3Index& kdi = view.build3dIndex(); for (PointId i = 0; i < view.size(); ++i) { // find the k-nearest neighbors auto ids = kdi.neighbors(i, m_knn); // compute covariance of the neighborhood auto B = eigen::computeCovariance(view, ids); // perform the eigen decomposition Eigen::SelfAdjointEigenSolver<Eigen::Matrix3f> solver(B); if (solver.info() != Eigen::Success) throwError("Cannot perform eigen decomposition."); auto eval = solver.eigenvalues(); Eigen::Vector3f normal = solver.eigenvectors().col(0); if (m_viewpointArg->set()) { PointRef p = view.point(i); Eigen::Vector3f vp( m_viewpoint.x - p.getFieldAs<double>(Dimension::Id::X), m_viewpoint.y - p.getFieldAs<double>(Dimension::Id::Y), m_viewpoint.z - p.getFieldAs<double>(Dimension::Id::Z)); if (vp.dot(normal) < 0) normal *= -1.0; } else if (m_up) { if (normal[2] < 0) normal *= -1.0; } view.setField(Dimension::Id::NormalX, i, normal[0]); view.setField(Dimension::Id::NormalY, i, normal[1]); view.setField(Dimension::Id::NormalZ, i, normal[2]); double sum = eval[0] + eval[1] + eval[2]; view.setField(Dimension::Id::Curvature, i, sum ? std::fabs(eval[0] / sum) : 0); } }