size_t
hstcpcli::request_buf_append(const char *start, const char *finish)
{
/*
if (num_req_sent > 0 || num_req_rcvd > 0) {
*/
if (num_req_rcvd > 0) {
close();
set_error(-1, "request_buf_append: protocol out of sync");
return 0;
}
const char *nl = start;
size_t num_req = 0;
while ((nl = memchr_char(nl, '\n', finish - nl))) {
if (nl == finish)
break;
num_req++;
nl++;
}
num_req++;
writebuf.append(start, finish);
if (*(finish - 1) != '\n')
writebuf.append_literal("\n");
num_req_bufd += num_req;
return num_req;
}
int
hstcpcli::request_send()
{
if (error_code < 0) {
return error_code;
}
clear_error();
if (fd.get() < 0) {
close();
return set_error(-1, "write: closed");
}
if (num_req_bufd == 0 || num_req_sent > 0 || num_req_rcvd > 0) {
close();
return set_error(-1, "request_send: protocol out of sync");
}
const size_t wrlen = writebuf.size();
const ssize_t r = send(fd.get(), writebuf.begin(), wrlen, MSG_NOSIGNAL);
if (r <= 0) {
close();
return set_error(-1, r < 0 ? "write: failed" : "write: eof");
}
writebuf.erase_front(r);
if (static_cast<size_t>(r) != wrlen) {
close();
return set_error(-1, "write: incomplete");
}
num_req_sent = num_req_bufd;
num_req_bufd = 0;
DBG(fprintf(stderr, "REQSEND 0\n"));
return 0;
}
void
ngram_search::xor_block(string_buffer& sb, unsigned int val)
{
for (string_buffer::iterator i = sb.begin(); i != sb.end(); ++i) {
(*i) ^= val;
}
}
const string_ref *
hstcpcli::get_next_row()
{
if (num_flds == 0) {
DBG(fprintf(stderr, "GNR NF 0\n"));
return 0;
}
if (flds.size() < num_flds) {
flds.resize(num_flds);
}
char *start = readbuf.begin() + cur_row_offset;
char *const finish = readbuf.begin() + response_end_offset - 1;
if (start >= finish) { /* start[0] == nl */
DBG(fprintf(stderr, "GNR FIN 0 %p %p\n", start, finish));
return 0;
}
for (size_t i = 0; i < num_flds; ++i) {
skip_one(start, finish);
char *const fld_begin = start;
read_token(start, finish);
char *const fld_end = start;
char *wp = fld_begin;
if (is_null_expression(fld_begin, fld_end)) {
/* null */
flds[i] = string_ref();
} else {
unescape_string(wp, fld_begin, fld_end); /* in-place */
flds[i] = string_ref(fld_begin, wp);
}
}
cur_row_offset = start - readbuf.begin();
return &flds[0];
}
void
hstcpcli::request_reset()
{
if (num_req_bufd) {
writebuf.erase_front(writebuf.size());
num_req_bufd = 0;
}
}
void
append_uint32(string_buffer& buf, uint32_t v)
{
char *const wp = buf.make_space(64);
const int len = snprintf(wp, 64, "%lu", static_cast<unsigned long>(v));
if (len > 0) {
buf.space_wrote(len);
}
}
void
hstcpcli::close()
{
fd.close();
readbuf.clear();
writebuf.clear();
response_end_offset = 0;
cur_row_offset = 0;
num_flds = 0;
num_req_bufd = 0;
num_req_sent = 0;
num_req_rcvd = 0;
}
int
hstcpcli::read_more()
{
const size_t block_size = 4096; // FIXME
char *const wp = readbuf.make_space(block_size);
const ssize_t rlen = read(fd.get(), wp, block_size);
if (rlen <= 0) {
if (rlen < 0) {
error_str = "read: failed";
} else {
error_str = "read: eof";
}
return rlen;
}
readbuf.space_wrote(rlen);
return rlen;
}
dtoken read_id() {
while (!eos() && m_curr_char != '(' && m_curr_char != ')' &&
m_curr_char != '#' && m_curr_char != ',' && (m_parsing_domains || m_curr_char != '.') &&
m_curr_char != ':' && m_curr_char != '=' && !iswspace(m_curr_char) ) {
save_and_next();
}
return m_reserved_symbols.string2dtoken(m_buffer.c_str());
}
void
ngram_search::serialize_block(string_buffer& sb, unsigned int tag,
const char *data, size_t len)
{
serialize_nat(sb, tag);
serialize_nat(sb, len);
sb.append(data, len);
}
// read an id of the form '|'.*'|'
dtoken read_bid() {
while (!eos() && m_curr_char != '|') {
save_and_next();
}
if (m_curr_char == '|') {
next();
}
return m_reserved_symbols.string2dtoken(m_buffer.c_str());
}
bool lookahead_newline() {
while (m_curr_char == ' ') {
save_and_next();
}
if (m_curr_char == '\n') {
next();
m_line++;
m_buffer.reset();
return true;
}
if (m_curr_char == '#') {
m_buffer.reset();
m_prev_char = 0;
read_comment();
return true;
}
return false;
}
int
hstcpcli::response_recv(size_t& num_flds_r)
{
if (error_code < 0) {
return error_code;
}
clear_error();
if (num_req_bufd > 0 || num_req_sent == 0 || num_req_rcvd > 0 ||
response_end_offset != 0) {
close();
return set_error(-1, "response_recv: protocol out of sync");
}
cur_row_offset = 0;
num_flds_r = num_flds = 0;
if (fd.get() < 0) {
return set_error(-1, "read: closed");
}
size_t offset = 0;
while (true) {
const char *const lbegin = readbuf.begin() + offset;
const char *const lend = readbuf.end();
const char *const nl = memchr_char(lbegin, '\n', lend - lbegin);
if (nl != 0) {
offset = (nl + 1) - readbuf.begin();
break;
}
if (read_more() <= 0) {
close();
return set_error(-1, "read: eof");
}
}
response_end_offset = offset;
--num_req_sent;
++num_req_rcvd;
char *start = readbuf.begin();
char *const finish = start + response_end_offset - 1;
const size_t resp_code = read_ui32(start, finish);
skip_one(start, finish);
num_flds_r = num_flds = read_ui32(start, finish);
if (resp_code != 0) {
skip_one(start, finish);
char *const err_begin = start;
read_token(start, finish);
char *const err_end = start;
std::string e = std::string(err_begin, err_end - err_begin);
if (e.empty()) {
e = "unknown_error";
}
return set_error(resp_code, e);
}
cur_row_offset = start - readbuf.begin();
DBG(fprintf(stderr, "[%s] ro=%zu eol=%zu\n",
std::string(readbuf.begin(), readbuf.begin() + response_end_offset)
.c_str(),
cur_row_offset, response_end_offset));
DBG(fprintf(stderr, "RES 0\n"));
return 0;
}
template <typename T> void
ngram_search::serialize_nat(string_buffer& sb, T value)
{
do {
const unsigned char b7 = value & 0x7fU;
value >>= 7;
const unsigned char cf = (value != 0) ? 0x80U : 0;
const unsigned char b = b7 | cf;
sb.append(reinterpret_cast<const char *>(&b), 1);
} while (value != 0);
}
void
hstcpcli::request_buf_auth(const char *secret, const char *typ)
{
if (num_req_sent > 0 || num_req_rcvd > 0) {
close();
set_error(-1, "request_buf_auth: protocol out of sync");
return;
}
if (typ == 0) {
typ = "1";
}
const string_ref typ_ref(typ, strlen(typ));
const string_ref secret_ref(secret, strlen(secret));
writebuf.append_literal("A\t");
writebuf.append(typ_ref.begin(), typ_ref.end());
writebuf.append_literal("\t");
writebuf.append(secret_ref.begin(), secret_ref.end());
writebuf.append_literal("\n");
++num_req_bufd;
}
void
hstcpcli::response_buf_remove()
{
if (response_end_offset == 0) {
close();
set_error(-1, "response_buf_remove: protocol out of sync");
return;
}
readbuf.erase_front(response_end_offset);
response_end_offset = 0;
--num_req_rcvd;
cur_row_offset = 0;
num_flds = 0;
}
int
hstcpcli::read_more()
{
const size_t block_size = 4096; // FIXME
char *const wp = readbuf.make_space(block_size);
int rlen;
errno = 0;
while ((rlen = read(fd.get(), wp, block_size)) <= 0) {
errno_buf = errno;
if (rlen < 0) {
if (errno == EINTR || errno == EAGAIN)
{
errno = 0;
continue;
}
error_str = String("read: failed", &my_charset_bin);
} else {
error_str = String("read: eof", &my_charset_bin);
}
return rlen;
}
readbuf.space_wrote(rlen);
return rlen;
}
bool
ngram_search::deserialize_block(const char *& start, const char *finish,
unsigned int& tag_r, string_buffer& data_r)
{
bool success = true;
size_t len = 0;
success &= deserialize_nat(start, finish, tag_r);
success &= deserialize_nat(start, finish, len);
const size_t cur_len = finish - start;
if (cur_len >= len) {
if (len != 0) {
data_r.append(start, len);
start += len;
}
} else {
success = false;
}
return success;
}
int
hstcpcli::get_result(hstresult& result)
{
/*
readbuf.swap(result.readbuf);
*/
char *const wp = result.readbuf.make_space(response_end_offset);
memcpy(wp, readbuf.begin(), response_end_offset);
result.readbuf.space_wrote(response_end_offset);
result.response_end_offset = response_end_offset;
result.num_flds = num_flds;
result.cur_row_offset = cur_row_offset;
if (result.flds.max_element < num_flds)
{
if (allocate_dynamic(&result.flds, num_flds))
return set_error(-1, "out of memory");
}
result.flds.elements = num_flds;
return 0;
}
append_string_buffer(string_buffer& b) : b(b) { b.reserve(100); }
const char * get_token_data() const {
return m_buffer.c_str();
}
dtoken next_token() {
for(;;) {
if (eos()) {
return TK_EOS;
}
m_buffer.reset();
switch (m_curr_char) {
case '#': // comment
read_comment();
break;
case '\n':
next();
m_line++;
return TK_NEWLINE;
case '\\':
// here we ignore a newline if it is preceded by a backslash.
// We need to take care, since anywhere else backshlash is used
// as a regular character
next();
save_char('\\');
if (lookahead_newline()) {
break;
}
return read_id();
case '(':
m_tok_pos = m_pos;
next();
return TK_LP;
case ')':
m_tok_pos = m_pos;
next();
return TK_RP;
case ',':
m_tok_pos = m_pos;
next();
return TK_COMMA;
case '=':
m_tok_pos = m_pos;
next();
return TK_EQ;
case '!':
m_tok_pos = m_pos;
next();
if(m_curr_char == '=') {
next();
return TK_NEQ;
}
return TK_NEG;
case ':':
m_tok_pos = m_pos;
next();
if (m_curr_char == '-') {
next();
return TK_LEFT_ARROW;
}
return TK_COLON;
case '\"':
return read_string();
case '|':
next();
return read_bid();
default:
if (iswspace(m_curr_char)) {
next();
break;
}
else if (iswdigit(m_curr_char)) {
m_tok_pos = m_pos;
save_and_next();
return read_num();
}
else {
char old = m_curr_char;
m_tok_pos = m_pos;
save_and_next();
if (old == '-' && iswdigit(m_curr_char)) {
return read_num();
}
else {
return read_id();
}
}
}
}
}
int
hstcpcli::response_recv(size_t& num_flds_r)
{
if (error_code < 0) {
return error_code;
}
clear_error();
if (num_req_bufd > 0 || num_req_sent == 0 || num_req_rcvd > 0 ||
response_end_offset != 0) {
close();
return set_error(-1, "response_recv: protocol out of sync");
}
cur_row_offset = 0;
num_flds_r = num_flds = 0;
if (fd.get() < 0) {
return set_error(-1, "read: closed");
}
size_t offset = 0;
while (true) {
const char *const lbegin = readbuf.begin() + offset;
const char *const lend = readbuf.end();
if (lbegin < lend)
{
const char *const nl = memchr_char(lbegin, '\n', lend - lbegin);
if (nl != 0) {
offset += (nl + 1) - lbegin;
break;
}
offset += lend - lbegin;
}
if (read_more() <= 0) {
close();
error_code = -1;
return error_code;
}
}
response_end_offset = offset;
--num_req_sent;
++num_req_rcvd;
char *start = readbuf.begin();
char *const finish = start + response_end_offset - 1;
const size_t resp_code = read_ui32(start, finish);
skip_one(start, finish);
num_flds_r = num_flds = read_ui32(start, finish);
if (resp_code != 0) {
skip_one(start, finish);
char *const err_begin = start;
read_token(start, finish);
char *const err_end = start;
String e = String(err_begin, err_end - err_begin, &my_charset_bin);
if (!e.length()) {
e = String("unknown_error", &my_charset_bin);
}
return set_error(resp_code, e);
}
cur_row_offset = start - readbuf.begin();
DBG(fprintf(stderr, "[%s] ro=%zu eol=%zu\n",
String(readbuf.begin(), readbuf.begin() + response_end_offset)
.c_str(),
cur_row_offset, response_end_offset));
DBG(fprintf(stderr, "RES 0\n"));
if (flds.max_element < num_flds)
{
if (allocate_dynamic(&flds, num_flds))
return set_error(-1, "out of memory");
}
flds.elements = num_flds;
return 0;
}
void
hstcpcli::request_buf_exec_generic(size_t pst_id, const string_ref& op,
const string_ref *kvs, size_t kvslen, uint32_t limit, uint32_t skip,
const string_ref& mod_op, const string_ref *mvs, size_t mvslen,
const hstcpcli_filter *fils, size_t filslen, int invalues_keypart,
const string_ref *invalues, size_t invalueslen)
{
if (num_req_sent > 0 || num_req_rcvd > 0) {
close();
set_error(-1, "request_buf_exec_generic: protocol out of sync");
return;
}
append_uint32(writebuf, pst_id); // FIXME size_t ?
writebuf.append_literal("\t");
writebuf.append(op.begin(), op.end());
writebuf.append_literal("\t");
append_uint32(writebuf, kvslen); // FIXME size_t ?
for (size_t i = 0; i < kvslen; ++i) {
const string_ref& kv = kvs[i];
append_delim_value(writebuf, kv.begin(), kv.end());
}
if (limit != 0 || skip != 0 || invalues_keypart >= 0 ||
mod_op.size() != 0 || filslen != 0) {
/* has more option */
writebuf.append_literal("\t");
append_uint32(writebuf, limit); // FIXME size_t ?
if (skip != 0 || invalues_keypart >= 0 ||
mod_op.size() != 0 || filslen != 0) {
writebuf.append_literal("\t");
append_uint32(writebuf, skip); // FIXME size_t ?
}
if (invalues_keypart >= 0) {
writebuf.append_literal("\t@\t");
append_uint32(writebuf, invalues_keypart);
writebuf.append_literal("\t");
append_uint32(writebuf, invalueslen);
for (size_t i = 0; i < invalueslen; ++i) {
const string_ref& s = invalues[i];
append_delim_value(writebuf, s.begin(), s.end());
}
}
for (size_t i = 0; i < filslen; ++i) {
const hstcpcli_filter& f = fils[i];
writebuf.append_literal("\t");
writebuf.append(f.filter_type.begin(), f.filter_type.end());
writebuf.append_literal("\t");
writebuf.append(f.op.begin(), f.op.end());
writebuf.append_literal("\t");
append_uint32(writebuf, f.ff_offset);
append_delim_value(writebuf, f.val.begin(), f.val.end());
}
if (mod_op.size() != 0) {
writebuf.append_literal("\t");
writebuf.append(mod_op.begin(), mod_op.end());
for (size_t i = 0; i < mvslen; ++i) {
const string_ref& mv = mvs[i];
append_delim_value(writebuf, mv.begin(), mv.end());
}
}
}
writebuf.append_literal("\n");
++num_req_bufd;
}
void
hstcpcli::request_buf_open_index(size_t pst_id, const char *dbn,
const char *tbl, const char *idx, const char *retflds, const char *filflds)
{
if (num_req_sent > 0 || num_req_rcvd > 0) {
close();
set_error(-1, "request_buf_open_index: protocol out of sync");
return;
}
const string_ref dbn_ref(dbn, strlen(dbn));
const string_ref tbl_ref(tbl, strlen(tbl));
const string_ref idx_ref(idx, strlen(idx));
const string_ref rfs_ref(retflds, strlen(retflds));
writebuf.append_literal("P\t");
append_uint32(writebuf, pst_id); // FIXME size_t ?
writebuf.append_literal("\t");
writebuf.append(dbn_ref.begin(), dbn_ref.end());
writebuf.append_literal("\t");
writebuf.append(tbl_ref.begin(), tbl_ref.end());
writebuf.append_literal("\t");
writebuf.append(idx_ref.begin(), idx_ref.end());
writebuf.append_literal("\t");
writebuf.append(rfs_ref.begin(), rfs_ref.end());
if (filflds != 0) {
const string_ref fls_ref(filflds, strlen(filflds));
writebuf.append_literal("\t");
writebuf.append(fls_ref.begin(), fls_ref.end());
}
writebuf.append_literal("\n");
++num_req_bufd;
}