Exemplo n.º 1
0
bool LeafNode::write_bucket(BlockWriter& writer, RecordBucket *bucket, Slice buffer)
{
    if (tree_->compressor_) {
        // 1. write to buffer
        Block block(buffer, 0, 0);
        BlockWriter wr(&block);
        if (!write_bucket(wr, bucket)) return false;

        // 2. compress
        assert(tree_->compressor_->max_compressed_length(block.size()) <=
            writer.remain());

        size_t compressed_length;
        if (!tree_->compressor_->compress(buffer.data(), block.size(),
             writer.addr(), &compressed_length)) {
            LOG_ERROR("compress msgbuf error, nid " << nid_);
            return false;
        }

        // 3. skip
        writer.skip(compressed_length);

        return true;
    } else {
        return write_bucket(writer, bucket);
    }
}
Exemplo n.º 2
0
bool InnerNode::write_msgbuf(BlockWriter& writer, MsgBuf *mb, Slice buffer)
{
    if (tree_->compressor_) {
        // 1. write to buffer
        Block block(buffer, 0, 0);
        BlockWriter wr(&block);
        if (!mb->write_to(wr)) return false;

        // 2. compress
        assert(tree_->compressor_->max_compressed_length(block.size()) <=
            writer.remain());

        size_t compressed_length;
        if (!tree_->compressor_->compress(buffer.data(), block.size(),
             writer.addr(), &compressed_length)) {
            LOG_ERROR("compress msgbuf error, nid " << nid_);
            return false;
        }

        // 3. skip
        writer.skip(compressed_length);

        return true;
    } else {
        return mb->write_to(writer);
    }
}
    /*
     * This method processes a block of rows in a single invocation.
     *
     * The inputs are retrieved via arg_reader
     * The outputs are returned via arg_writer
     */
    virtual void processBlock(ServerInterface &srvInterface,
                              BlockReader &arg_reader,
                              BlockWriter &res_writer)
    {

        // While we have inputs to process
        do {
            if (arg_reader.getStringRef(0).isNull()) {
                res_writer.getStringRef().setNull();
            }
            else {

                string data = arg_reader.getStringRef(0).str();
                string path = arg_reader.getStringRef(1).str();
                ResponseInfo response = XMLFunctions::xpath(data, path, srvInterface);
                if ((int) response.size() < arg_reader.getIntRef(2)) {
                    res_writer.getStringRef().setNull();
                } else if ((int) response.size() > 0) {
                    res_writer.getStringRef().copy( response[arg_reader.getIntRef(2)-1]);
                } else {
                    res_writer.getStringRef().setNull();
                }
            }

            res_writer.next();
        } while (arg_reader.next());
    }
Exemplo n.º 4
0
bool Layout::write_block_meta(BlockMeta* meta, BlockWriter& writer)
{
    if (!writer.writeUInt64(meta->offset)) return false;
    if (!writer.writeUInt32(meta->skeleton_size)) return false;
    if (!writer.writeUInt32(meta->total_size)) return false;
    if (!writer.writeUInt16(meta->crc)) return false;
    return true;
}
Exemplo n.º 5
0
bool SchemaNode::write_to(BlockWriter& writer, size_t& skeleton_size)
{
    if (!writer.writeUInt64(root_node_id)) return false;
    if (!writer.writeUInt64(next_inner_node_id)) return false;
    if (!writer.writeUInt64(next_leaf_node_id)) return false;
    if (!writer.writeUInt64(tree_depth)) return false;
    skeleton_size = SCHEMA_NODE_SIZE;
    return true;
}
Exemplo n.º 6
0
bool Msg::write_to(BlockWriter& writer)
{
    if (!writer.writeUInt8((uint8_t)type)) return false;
    if (!writer.writeSlice(key)) return false;
    if (type == Put) {
        if (!writer.writeSlice(value)) return false;
    }
    return true;
}
Exemplo n.º 7
0
bool Layout::write_index(BlockWriter& writer)
{
    ScopedMutex block_index_lock(&block_index_mtx_);
    if (!writer.writeUInt32(block_index_.size())) return false;
    for(map<bid_t, BlockMeta*>::iterator it = block_index_.begin();
        it != block_index_.end(); it++ ) {
        if (!writer.writeUInt64(it->first)) return false;
        if (!write_block_meta(it->second, writer)) return false;
    }
    return true;
}
Exemplo n.º 8
0
bool LeafNode::write_buckets_info(BlockWriter& writer)
{
    if (!writer.writeUInt32(buckets_info_.size())) return false;
    for (size_t i = 0; i < buckets_info_.size(); i++ ) {
        if (!writer.writeSlice(buckets_info_[i].key)) return false;
        if (!writer.writeUInt32(buckets_info_[i].offset)) return false;
        if (!writer.writeUInt32(buckets_info_[i].length)) return false;
        if (!writer.writeUInt32(buckets_info_[i].uncompressed_length)) return false;
    }
    return true;
}
Exemplo n.º 9
0
bool Layout::write_superblock(BlockWriter& writer)
{
    if (!writer.writeUInt64(superblock_->magic_number0)) return false;
    if (!writer.writeUInt8(superblock_->major_version)) return false;
    if (!writer.writeUInt8(superblock_->minor_version)) return false;

    if (superblock_->index_block_meta) {
        if (!writer.writeBool(true)) return false;
        if (!write_block_meta(superblock_->index_block_meta, writer)) return false;
    } else {
        if (!writer.writeBool(false)) return false;
    }

    if (!writer.writeUInt64(superblock_->magic_number1)) return false;
    return true;
}
Exemplo n.º 10
0
bool LeafNode::write_bucket(BlockWriter& writer, RecordBucket *bucket)
{
    if (!writer.writeUInt32(bucket->size())) return false;
    for (size_t j = 0; j < bucket->size(); j++) {
        if (!(*bucket)[j].write_to(writer)) return false;
    }
    return true;
}
Exemplo n.º 11
0
bool LeafNode::write_to(BlockWriter& writer, size_t& skeleton_size)
{
    assert(status_ == kNew || status_ == kFullLoaded);

    size_t skeleton_pos = writer.pos();
    skeleton_size = 8 + 8 + buckets_info_size_;
    if (!writer.skip(skeleton_size)) return false;

    Slice buffer;
    if (tree_->compressor_) {
        size_t buffer_length = 0;
        for (size_t i = 0; i < records_.buckets_number(); i++) {
            if (buffer_length < records_.bucket_length(i)) {
                buffer_length = records_.bucket_length(i);
            }
        }
        buffer = Slice::alloc(buffer_length);
    }

    assert(records_.buckets_number() == buckets_info_.size());
    for (size_t i = 0; i < records_.buckets_number(); i++ ) {
        RecordBucket* bucket = records_.bucket(i);

        buckets_info_[i].offset = writer.pos();
        if (!write_bucket(writer, bucket, buffer)) {
            if (buffer.size()) {
                buffer.destroy();
            }
            return false;
        }
        buckets_info_[i].length = writer.pos() - buckets_info_[i].offset;
        buckets_info_[i].uncompressed_length = records_.bucket_length(i);
    }
    size_t last_pos = writer.pos();

    if (buffer.size()) {
        buffer.destroy();
    }

    writer.seek(skeleton_pos);
    if (!writer.writeUInt64(left_sibling_)) return false;
    if (!writer.writeUInt64(right_sibling_)) return false;

    if (!write_buckets_info(writer)) {
        LOG_ERROR("write buckets info error, nid " << nid_);
        return false;
    }
    writer.seek(last_pos);
    return true;
}
Exemplo n.º 12
0
			virtual void processBlock(ServerInterface &srvInterface,
				            BlockReader &arg_reader,
					        BlockWriter &res_writer) {
				if (arg_reader.getNumCols() != 1)
					vt_report_error(0, "Function only accept 1 arguments, but %zu provided", arg_reader.getNumCols());

				// While we have inputs to process
				do {
					const vint num = arg_reader.getIntRef(0);

					// check for valid month number
					if (num > BILLION)
					vt_report_error(0, "ERROR: OUT OF RANGE");

					res_writer.getStringRef().copy(spell_out(num));
					res_writer.next();
				} while (arg_reader.next());					
			}
Exemplo n.º 13
0
bool MsgBuf::write_to(BlockWriter& writer)
{
    if (!writer.writeUInt32(container_.size())) return false;
    for (ContainerType::iterator it = container_.begin();
        it != container_.end(); it ++ ) {
        if (!it->write_to(writer)) return false;
    }
    return true;
}
    /*
     * This method processes a block of rows in a single invocation.
     *
     * The inputs are retrieved via arg_reader
     * The outputs are returned via arg_writer
     */
    virtual void processBlock(ServerInterface &srvInterface,
                              BlockReader &arg_reader,
                              BlockWriter &res_writer)
    {
        // Basic error checking
        if (arg_reader.getNumCols() != 2)
            vt_report_error(0, "Function only accept 2 arguments, but %zu provided",
                            arg_reader.getNumCols());

        // While we have inputs to process
        do {
            // Get a copy of the input string
            const std::string& patternStr = arg_reader.getStringRef(0).str();
            const std::string& inStr = arg_reader.getStringRef(1).str();

            size_t m = patternStr.size();
            size_t n = inStr.size();
            size_t d[m+1][n+1];
            for (size_t i = 0; i <= m; ++i)
                d[i][0] = i; // the distance of any first string to an empty second string
            for (size_t j = 0; j <= n; ++j)
                d[0][j] = j; // the distance of any second string to an empty first string

            for (size_t j = 1; j <= n; ++j)
                for (size_t i = 1; i <= m; ++i)
                {
                    if (patternStr[i-1] == inStr[j-1])
                        d[i][j] = d[i-1][j-1];       // no operation required
                    else
                    {
                        d[i][j] = std::min(std::min(d[i-1][j] + 1,  // a deletion
                                                    d[i][j-1] + 1),  // an insertion
                                           d[i-1][j-1] + 1); // a substitution
                    }
                }

            res_writer.setInt(d[m][n]);
            res_writer.next();
        } while (arg_reader.next());
    }
    /*
     * This method processes a block of rows in a single invocation.
     *
     * The inputs are retrieved via arg_reader
     * The outputs are returned via arg_writer
     */
    virtual void processBlock(ServerInterface &srvInterface,
                              BlockReader &arg_reader,
                              BlockWriter &res_writer)
    {

        // While we have inputs to process
        do {
            if (arg_reader.getStringRef(0).isNull()) {
                res_writer.getStringRef().setNull();
            }
            else {

                string data = arg_reader.getStringRef(0).str();
                string style = arg_reader.getStringRef(1).str();
                res_writer.getStringRef().copy(
                    XMLFunctions::xslt(data, style, srvInterface)
                );
            }

            res_writer.next();
        } while (arg_reader.next());
    }
Exemplo n.º 16
0
void GeometryGenerator::Execute(bool skipNonIntersectingBlocks)
		throw (GenerationError) {

	if (skipNonIntersectingBlocks) {
		throw GenerationErrorMessage(
				"Skip non intersecting blocks functionality currently not available. See ticket #651");
	}

	this->PreExecute();
	double bounds[6];
	this->ComputeBounds(bounds);
	Domain domain(this->OriginWorking, this->SiteCounts);

	GeometryWriter writer(this->OutputGeometryFile, domain.GetBlockSize(),
			domain.GetBlockCounts());

	for (BlockIterator blockIt = domain.begin(); blockIt != domain.end();
			++blockIt) {
		// Open the BlockStarted context of the writer; this will
		// deal with flushing the state to the file (or not, in the
		// case where there are no fluid sites).
		BlockWriter* blockWriterPtr = writer.StartNextBlock();
		Block& block = *blockIt;

		int side = 0; // represents whether the block is inside (-1) outside (+1) or undetermined (0)

		if (skipNonIntersectingBlocks) {
			side = this->BlockInsideOrOutsideSurface(block);
		} else { // don't use the optimisation -- check every site
			side = 0;
		}

		switch (side) {
		case 1:
			// Block is entirely outside the domain.
			// We don't have to do anything.
			break;
		case 0:
			// Block has some surface within it.
			for (SiteIterator siteIt = block.begin(); siteIt != block.end();
					++siteIt) {
				Site& site = **siteIt;
				this->ClassifySite(site);
				// here we should check site
				if (site.IsFluid) {
					blockWriterPtr->IncrementFluidSitesCount();
					WriteFluidSite(*blockWriterPtr, site);
				} else {
					WriteSolidSite(*blockWriterPtr, site);
				}

			}
			break;
		case -1:
			// Block is entirely inside the domain
			for (SiteIterator siteIt = block.begin(); siteIt != block.end();
					++siteIt) {
				Site& site = **siteIt;
				site.IsFluidKnown = true;
				site.IsFluid = true;
				site.CreateLinksVector();
				for (unsigned int link_index = 0;
						link_index < site.Links.size(); ++link_index) {
					site.Links[link_index].Type = geometry::CUT_NONE;
				}
				blockWriterPtr->IncrementFluidSitesCount();
				WriteFluidSite(*blockWriterPtr, site);
			}
			break;
		default:
			break;
		}
		blockWriterPtr->Finish();
		blockWriterPtr->Write(writer);
		delete blockWriterPtr;
	}
	writer.Close();
}
    /*
     * This method processes a block of rows in a single invocation.
     *
     * The inputs are retrieved via arg_reader
     * The outputs are returned via arg_writer
     */
    virtual void processBlock(ServerInterface &srvInterface,
                              BlockReader &arg_reader,
                              BlockWriter &res_writer)
    {
        // Basic error checking
        if (arg_reader.getNumCols() != 4)
            vt_report_error(0, "Function only accept 1 arguments, but %zu provided", 
                            arg_reader.getNumCols());
        
        // While we have inputs to process
        do {

            char curlresponse[1024]; 

            // Get a copy of the input string
            std::string  streetStr = arg_reader.getStringRef(0).str();
            std::string  cityStr = arg_reader.getStringRef(1).str();
            std::string  stateStr = arg_reader.getStringRef(2).str();
            std::string  zipStr = arg_reader.getStringRef(3).str();

	    // Replaces spaces with %20 for URLs
            for (size_t pos = streetStr.find(' '); 
		 pos != string::npos;
		 pos = streetStr.find(' ',pos)) 
	    {
	      	streetStr.replace(pos,1,"%20");
	    } 

  	     for (size_t pos = cityStr.find(' ');
                 pos != string::npos;
                 pos = cityStr.find(' ',pos))                         
            {
                cityStr.replace(pos,1,"%20");
            }

            // Build the URL 
            std::string URL = baseURL + "&streetAddress=" + streetStr;
            URL += "&city=" + cityStr;
            URL += "&state=" + stateStr;
            URL += "&zip=" + zipStr;

            if(curl) {
              curl_easy_setopt(curl, CURLOPT_URL, URL.c_str());
            }
    
	    // curl setup
            curl_easy_setopt(curl, CURLOPT_HEADER, 0);
            curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
            curl_easy_setopt(curl, CURLOPT_WRITEDATA, &curlresponse );

	    // perform the webservice call	
            res = curl_easy_perform(curl);

            // parse webservice response to get the latitude and longitude data 
            std::vector<std::string> tokens;
            split(tokens,string(curlresponse),',');
            std::string latlong = tokens[3] + "," + tokens[4];

	    // copy data back to Vertica
            res_writer.getStringRef().copy(latlong);
            res_writer.next();
        } while (arg_reader.next());
    }
Exemplo n.º 18
0
bool InnerNode::write_to(BlockWriter& writer, size_t& skeleton_size)
{
    // get length of skeleton and reserve space for skeleton
    size_t skeleton_offset = writer.pos();
    size_t skeleton_length = 1 + 4 + 8 + 4 + 4 + 4;
    for (size_t i = 0; i < pivots_.size(); i++) {
        skeleton_length += pivot_size(pivots_[i].key);
    }
    if (!writer.skip(skeleton_length)) return false;

    // prepare buffer if compression is enabled
    Slice buffer;
    if (tree_->compressor_) {
        // get buffer length to serialize msgbuf
        size_t buffer_length = first_msgbuf_->size();
        for (size_t i = 0; i < pivots_.size(); i++) {
            if (pivots_[i].msgbuf->size() > buffer_length)
                buffer_length = pivots_[i].msgbuf->size();
        }

        buffer = Slice::alloc(buffer_length);
    }

    // write the first msgbuf
    first_msgbuf_offset_ = writer.pos();
    if (!write_msgbuf(writer, first_msgbuf_, buffer)) return false;
    first_msgbuf_length_ = writer.pos() - first_msgbuf_offset_;
    first_msgbuf_uncompressed_length_ = first_msgbuf_->size();

    // write rest msgbufs
    for (size_t i = 0; i < pivots_.size(); i++) {
        pivots_[i].offset = writer.pos();
        if (!write_msgbuf(writer, pivots_[i].msgbuf, buffer)) return false;
        pivots_[i].length = writer.pos() - pivots_[i].offset;
        pivots_[i].uncompressed_length = pivots_[i].msgbuf->size();
    }

    if (buffer.size()) {
        buffer.destroy();
    }

    size_t last_offset = writer.pos();

    // seek to the head and write index
    writer.seek(skeleton_offset);
    if (!writer.writeBool(bottom_)) return false;
    if (!writer.writeUInt32(pivots_.size())) return false;

    if (!writer.writeUInt64(first_child_)) return false;
    if (!writer.writeUInt32(first_msgbuf_offset_)) return false;
    if (!writer.writeUInt32(first_msgbuf_length_)) return false;
    if (!writer.writeUInt32(first_msgbuf_uncompressed_length_)) return false;

    for (size_t i = 0; i < pivots_.size(); i++) {
        if (!writer.writeSlice(pivots_[i].key)) return false;
        if (!writer.writeUInt64(pivots_[i].child)) return false;
        if (!writer.writeUInt32(pivots_[i].offset)) return false;
        if (!writer.writeUInt32(pivots_[i].length)) return false;
        if (!writer.writeUInt32(pivots_[i].uncompressed_length)) return false;
    }

    writer.seek(last_offset);
    skeleton_size = skeleton_length;
    return true;
}