bool LeafNode::write_bucket(BlockWriter& writer, RecordBucket *bucket, Slice buffer)
{
if (tree_->compressor_) {
// 1. write to buffer
Block block(buffer, 0, 0);
BlockWriter wr(&block);
if (!write_bucket(wr, bucket)) return false;
// 2. compress
assert(tree_->compressor_->max_compressed_length(block.size()) <=
writer.remain());
size_t compressed_length;
if (!tree_->compressor_->compress(buffer.data(), block.size(),
writer.addr(), &compressed_length)) {
LOG_ERROR("compress msgbuf error, nid " << nid_);
return false;
}
// 3. skip
writer.skip(compressed_length);
return true;
} else {
return write_bucket(writer, bucket);
}
}
bool InnerNode::write_msgbuf(BlockWriter& writer, MsgBuf *mb, Slice buffer)
{
if (tree_->compressor_) {
// 1. write to buffer
Block block(buffer, 0, 0);
BlockWriter wr(&block);
if (!mb->write_to(wr)) return false;
// 2. compress
assert(tree_->compressor_->max_compressed_length(block.size()) <=
writer.remain());
size_t compressed_length;
if (!tree_->compressor_->compress(buffer.data(), block.size(),
writer.addr(), &compressed_length)) {
LOG_ERROR("compress msgbuf error, nid " << nid_);
return false;
}
// 3. skip
writer.skip(compressed_length);
return true;
} else {
return mb->write_to(writer);
}
}
/*
* This method processes a block of rows in a single invocation.
*
* The inputs are retrieved via arg_reader
* The outputs are returned via arg_writer
*/
virtual void processBlock(ServerInterface &srvInterface,
BlockReader &arg_reader,
BlockWriter &res_writer)
{
// While we have inputs to process
do {
if (arg_reader.getStringRef(0).isNull()) {
res_writer.getStringRef().setNull();
}
else {
string data = arg_reader.getStringRef(0).str();
string path = arg_reader.getStringRef(1).str();
ResponseInfo response = XMLFunctions::xpath(data, path, srvInterface);
if ((int) response.size() < arg_reader.getIntRef(2)) {
res_writer.getStringRef().setNull();
} else if ((int) response.size() > 0) {
res_writer.getStringRef().copy( response[arg_reader.getIntRef(2)-1]);
} else {
res_writer.getStringRef().setNull();
}
}
res_writer.next();
} while (arg_reader.next());
}
bool Layout::write_block_meta(BlockMeta* meta, BlockWriter& writer)
{
if (!writer.writeUInt64(meta->offset)) return false;
if (!writer.writeUInt32(meta->skeleton_size)) return false;
if (!writer.writeUInt32(meta->total_size)) return false;
if (!writer.writeUInt16(meta->crc)) return false;
return true;
}
bool SchemaNode::write_to(BlockWriter& writer, size_t& skeleton_size)
{
if (!writer.writeUInt64(root_node_id)) return false;
if (!writer.writeUInt64(next_inner_node_id)) return false;
if (!writer.writeUInt64(next_leaf_node_id)) return false;
if (!writer.writeUInt64(tree_depth)) return false;
skeleton_size = SCHEMA_NODE_SIZE;
return true;
}
bool Msg::write_to(BlockWriter& writer)
{
if (!writer.writeUInt8((uint8_t)type)) return false;
if (!writer.writeSlice(key)) return false;
if (type == Put) {
if (!writer.writeSlice(value)) return false;
}
return true;
}
bool Layout::write_index(BlockWriter& writer)
{
ScopedMutex block_index_lock(&block_index_mtx_);
if (!writer.writeUInt32(block_index_.size())) return false;
for(map<bid_t, BlockMeta*>::iterator it = block_index_.begin();
it != block_index_.end(); it++ ) {
if (!writer.writeUInt64(it->first)) return false;
if (!write_block_meta(it->second, writer)) return false;
}
return true;
}
bool LeafNode::write_buckets_info(BlockWriter& writer)
{
if (!writer.writeUInt32(buckets_info_.size())) return false;
for (size_t i = 0; i < buckets_info_.size(); i++ ) {
if (!writer.writeSlice(buckets_info_[i].key)) return false;
if (!writer.writeUInt32(buckets_info_[i].offset)) return false;
if (!writer.writeUInt32(buckets_info_[i].length)) return false;
if (!writer.writeUInt32(buckets_info_[i].uncompressed_length)) return false;
}
return true;
}
bool Layout::write_superblock(BlockWriter& writer)
{
if (!writer.writeUInt64(superblock_->magic_number0)) return false;
if (!writer.writeUInt8(superblock_->major_version)) return false;
if (!writer.writeUInt8(superblock_->minor_version)) return false;
if (superblock_->index_block_meta) {
if (!writer.writeBool(true)) return false;
if (!write_block_meta(superblock_->index_block_meta, writer)) return false;
} else {
if (!writer.writeBool(false)) return false;
}
if (!writer.writeUInt64(superblock_->magic_number1)) return false;
return true;
}
bool LeafNode::write_bucket(BlockWriter& writer, RecordBucket *bucket)
{
if (!writer.writeUInt32(bucket->size())) return false;
for (size_t j = 0; j < bucket->size(); j++) {
if (!(*bucket)[j].write_to(writer)) return false;
}
return true;
}
bool LeafNode::write_to(BlockWriter& writer, size_t& skeleton_size)
{
assert(status_ == kNew || status_ == kFullLoaded);
size_t skeleton_pos = writer.pos();
skeleton_size = 8 + 8 + buckets_info_size_;
if (!writer.skip(skeleton_size)) return false;
Slice buffer;
if (tree_->compressor_) {
size_t buffer_length = 0;
for (size_t i = 0; i < records_.buckets_number(); i++) {
if (buffer_length < records_.bucket_length(i)) {
buffer_length = records_.bucket_length(i);
}
}
buffer = Slice::alloc(buffer_length);
}
assert(records_.buckets_number() == buckets_info_.size());
for (size_t i = 0; i < records_.buckets_number(); i++ ) {
RecordBucket* bucket = records_.bucket(i);
buckets_info_[i].offset = writer.pos();
if (!write_bucket(writer, bucket, buffer)) {
if (buffer.size()) {
buffer.destroy();
}
return false;
}
buckets_info_[i].length = writer.pos() - buckets_info_[i].offset;
buckets_info_[i].uncompressed_length = records_.bucket_length(i);
}
size_t last_pos = writer.pos();
if (buffer.size()) {
buffer.destroy();
}
writer.seek(skeleton_pos);
if (!writer.writeUInt64(left_sibling_)) return false;
if (!writer.writeUInt64(right_sibling_)) return false;
if (!write_buckets_info(writer)) {
LOG_ERROR("write buckets info error, nid " << nid_);
return false;
}
writer.seek(last_pos);
return true;
}
virtual void processBlock(ServerInterface &srvInterface,
BlockReader &arg_reader,
BlockWriter &res_writer) {
if (arg_reader.getNumCols() != 1)
vt_report_error(0, "Function only accept 1 arguments, but %zu provided", arg_reader.getNumCols());
// While we have inputs to process
do {
const vint num = arg_reader.getIntRef(0);
// check for valid month number
if (num > BILLION)
vt_report_error(0, "ERROR: OUT OF RANGE");
res_writer.getStringRef().copy(spell_out(num));
res_writer.next();
} while (arg_reader.next());
}
bool MsgBuf::write_to(BlockWriter& writer)
{
if (!writer.writeUInt32(container_.size())) return false;
for (ContainerType::iterator it = container_.begin();
it != container_.end(); it ++ ) {
if (!it->write_to(writer)) return false;
}
return true;
}
/*
* This method processes a block of rows in a single invocation.
*
* The inputs are retrieved via arg_reader
* The outputs are returned via arg_writer
*/
virtual void processBlock(ServerInterface &srvInterface,
BlockReader &arg_reader,
BlockWriter &res_writer)
{
// Basic error checking
if (arg_reader.getNumCols() != 2)
vt_report_error(0, "Function only accept 2 arguments, but %zu provided",
arg_reader.getNumCols());
// While we have inputs to process
do {
// Get a copy of the input string
const std::string& patternStr = arg_reader.getStringRef(0).str();
const std::string& inStr = arg_reader.getStringRef(1).str();
size_t m = patternStr.size();
size_t n = inStr.size();
size_t d[m+1][n+1];
for (size_t i = 0; i <= m; ++i)
d[i][0] = i; // the distance of any first string to an empty second string
for (size_t j = 0; j <= n; ++j)
d[0][j] = j; // the distance of any second string to an empty first string
for (size_t j = 1; j <= n; ++j)
for (size_t i = 1; i <= m; ++i)
{
if (patternStr[i-1] == inStr[j-1])
d[i][j] = d[i-1][j-1]; // no operation required
else
{
d[i][j] = std::min(std::min(d[i-1][j] + 1, // a deletion
d[i][j-1] + 1), // an insertion
d[i-1][j-1] + 1); // a substitution
}
}
res_writer.setInt(d[m][n]);
res_writer.next();
} while (arg_reader.next());
}
/*
* This method processes a block of rows in a single invocation.
*
* The inputs are retrieved via arg_reader
* The outputs are returned via arg_writer
*/
virtual void processBlock(ServerInterface &srvInterface,
BlockReader &arg_reader,
BlockWriter &res_writer)
{
// While we have inputs to process
do {
if (arg_reader.getStringRef(0).isNull()) {
res_writer.getStringRef().setNull();
}
else {
string data = arg_reader.getStringRef(0).str();
string style = arg_reader.getStringRef(1).str();
res_writer.getStringRef().copy(
XMLFunctions::xslt(data, style, srvInterface)
);
}
res_writer.next();
} while (arg_reader.next());
}
void GeometryGenerator::Execute(bool skipNonIntersectingBlocks)
throw (GenerationError) {
if (skipNonIntersectingBlocks) {
throw GenerationErrorMessage(
"Skip non intersecting blocks functionality currently not available. See ticket #651");
}
this->PreExecute();
double bounds[6];
this->ComputeBounds(bounds);
Domain domain(this->OriginWorking, this->SiteCounts);
GeometryWriter writer(this->OutputGeometryFile, domain.GetBlockSize(),
domain.GetBlockCounts());
for (BlockIterator blockIt = domain.begin(); blockIt != domain.end();
++blockIt) {
// Open the BlockStarted context of the writer; this will
// deal with flushing the state to the file (or not, in the
// case where there are no fluid sites).
BlockWriter* blockWriterPtr = writer.StartNextBlock();
Block& block = *blockIt;
int side = 0; // represents whether the block is inside (-1) outside (+1) or undetermined (0)
if (skipNonIntersectingBlocks) {
side = this->BlockInsideOrOutsideSurface(block);
} else { // don't use the optimisation -- check every site
side = 0;
}
switch (side) {
case 1:
// Block is entirely outside the domain.
// We don't have to do anything.
break;
case 0:
// Block has some surface within it.
for (SiteIterator siteIt = block.begin(); siteIt != block.end();
++siteIt) {
Site& site = **siteIt;
this->ClassifySite(site);
// here we should check site
if (site.IsFluid) {
blockWriterPtr->IncrementFluidSitesCount();
WriteFluidSite(*blockWriterPtr, site);
} else {
WriteSolidSite(*blockWriterPtr, site);
}
}
break;
case -1:
// Block is entirely inside the domain
for (SiteIterator siteIt = block.begin(); siteIt != block.end();
++siteIt) {
Site& site = **siteIt;
site.IsFluidKnown = true;
site.IsFluid = true;
site.CreateLinksVector();
for (unsigned int link_index = 0;
link_index < site.Links.size(); ++link_index) {
site.Links[link_index].Type = geometry::CUT_NONE;
}
blockWriterPtr->IncrementFluidSitesCount();
WriteFluidSite(*blockWriterPtr, site);
}
break;
default:
break;
}
blockWriterPtr->Finish();
blockWriterPtr->Write(writer);
delete blockWriterPtr;
}
writer.Close();
}
/*
* This method processes a block of rows in a single invocation.
*
* The inputs are retrieved via arg_reader
* The outputs are returned via arg_writer
*/
virtual void processBlock(ServerInterface &srvInterface,
BlockReader &arg_reader,
BlockWriter &res_writer)
{
// Basic error checking
if (arg_reader.getNumCols() != 4)
vt_report_error(0, "Function only accept 1 arguments, but %zu provided",
arg_reader.getNumCols());
// While we have inputs to process
do {
char curlresponse[1024];
// Get a copy of the input string
std::string streetStr = arg_reader.getStringRef(0).str();
std::string cityStr = arg_reader.getStringRef(1).str();
std::string stateStr = arg_reader.getStringRef(2).str();
std::string zipStr = arg_reader.getStringRef(3).str();
// Replaces spaces with %20 for URLs
for (size_t pos = streetStr.find(' ');
pos != string::npos;
pos = streetStr.find(' ',pos))
{
streetStr.replace(pos,1,"%20");
}
for (size_t pos = cityStr.find(' ');
pos != string::npos;
pos = cityStr.find(' ',pos))
{
cityStr.replace(pos,1,"%20");
}
// Build the URL
std::string URL = baseURL + "&streetAddress=" + streetStr;
URL += "&city=" + cityStr;
URL += "&state=" + stateStr;
URL += "&zip=" + zipStr;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, URL.c_str());
}
// curl setup
curl_easy_setopt(curl, CURLOPT_HEADER, 0);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &curlresponse );
// perform the webservice call
res = curl_easy_perform(curl);
// parse webservice response to get the latitude and longitude data
std::vector<std::string> tokens;
split(tokens,string(curlresponse),',');
std::string latlong = tokens[3] + "," + tokens[4];
// copy data back to Vertica
res_writer.getStringRef().copy(latlong);
res_writer.next();
} while (arg_reader.next());
}
bool InnerNode::write_to(BlockWriter& writer, size_t& skeleton_size)
{
// get length of skeleton and reserve space for skeleton
size_t skeleton_offset = writer.pos();
size_t skeleton_length = 1 + 4 + 8 + 4 + 4 + 4;
for (size_t i = 0; i < pivots_.size(); i++) {
skeleton_length += pivot_size(pivots_[i].key);
}
if (!writer.skip(skeleton_length)) return false;
// prepare buffer if compression is enabled
Slice buffer;
if (tree_->compressor_) {
// get buffer length to serialize msgbuf
size_t buffer_length = first_msgbuf_->size();
for (size_t i = 0; i < pivots_.size(); i++) {
if (pivots_[i].msgbuf->size() > buffer_length)
buffer_length = pivots_[i].msgbuf->size();
}
buffer = Slice::alloc(buffer_length);
}
// write the first msgbuf
first_msgbuf_offset_ = writer.pos();
if (!write_msgbuf(writer, first_msgbuf_, buffer)) return false;
first_msgbuf_length_ = writer.pos() - first_msgbuf_offset_;
first_msgbuf_uncompressed_length_ = first_msgbuf_->size();
// write rest msgbufs
for (size_t i = 0; i < pivots_.size(); i++) {
pivots_[i].offset = writer.pos();
if (!write_msgbuf(writer, pivots_[i].msgbuf, buffer)) return false;
pivots_[i].length = writer.pos() - pivots_[i].offset;
pivots_[i].uncompressed_length = pivots_[i].msgbuf->size();
}
if (buffer.size()) {
buffer.destroy();
}
size_t last_offset = writer.pos();
// seek to the head and write index
writer.seek(skeleton_offset);
if (!writer.writeBool(bottom_)) return false;
if (!writer.writeUInt32(pivots_.size())) return false;
if (!writer.writeUInt64(first_child_)) return false;
if (!writer.writeUInt32(first_msgbuf_offset_)) return false;
if (!writer.writeUInt32(first_msgbuf_length_)) return false;
if (!writer.writeUInt32(first_msgbuf_uncompressed_length_)) return false;
for (size_t i = 0; i < pivots_.size(); i++) {
if (!writer.writeSlice(pivots_[i].key)) return false;
if (!writer.writeUInt64(pivots_[i].child)) return false;
if (!writer.writeUInt32(pivots_[i].offset)) return false;
if (!writer.writeUInt32(pivots_[i].length)) return false;
if (!writer.writeUInt32(pivots_[i].uncompressed_length)) return false;
}
writer.seek(last_offset);
skeleton_size = skeleton_length;
return true;
}
