_RIBS_INLINE_ int read_data(struct timeout_handler *timeout_handler, int cfd, struct vmbuf *response, size_t ofs) {
int res = vmbuf_read(response, cfd);
if (0 > res) {
close(cfd);
return -1;
}
*vmbuf_wloc(response) = 0;
char *data, *line;
/* Keep reading if
* 1.you don't see a CRLF
* OR
* 2.see a dash and CRLF in the current line
*/
while ((NULL == strstr((data = vmbuf_data_ofs(response, ofs)), CRLF))
|| ((NULL != strchr((line = strrstr(data, CRLF)), '-')) && (NULL != strstr(line, CRLF)))) {
if (1 == res)
sendemail_yield(timeout_handler, cfd);
if (0 >= (res = vmbuf_read(response, cfd))) {
LOGGER_PERROR("read");
close(cfd);
return -1;
}
}
*vmbuf_wloc(response) = 0;
return 0;
}
int vmbuf_inflate2(struct vmbuf *inbuf, struct vmbuf *outbuf) {
z_stream strm;
_init_alloc(&strm);
if (Z_OK != inflateInit2(&strm, 15+16))
return -1;
for (;;)
{
strm.next_in = (uint8_t *)vmbuf_rloc(inbuf);
strm.avail_in = vmbuf_ravail(inbuf);
if (0 == strm.avail_in)
break;
vmbuf_resize_if_less(outbuf, strm.avail_in << 1);
strm.next_out = (uint8_t *)vmbuf_wloc(outbuf);
strm.avail_out = vmbuf_wavail(outbuf);
int res = inflate(&strm, Z_NO_FLUSH);
vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out);
if (res == Z_STREAM_END)
break;
if (Z_OK != res)
return inflateEnd(&strm), -1;
vmbuf_rseek(inbuf, vmbuf_ravail(inbuf) - strm.avail_in);
}
inflateEnd(&strm);
return 0;
}
static void
trigger_writer (
const char *filename,
struct logz_file_def *filedef) {
vmbuf_reset(&write_buffer);
ssize_t res;
char *fn = basename(ribs_strdup(filename));
while(1) {
vmbuf_reset(&write_buffer);
res = read(filedef->fd, vmbuf_wloc(&write_buffer), (BUFSIZ + 1024) &~ 1024);
filedef->size += res;
lseek (filedef->fd, filedef->size, SEEK_SET);
if (0 > vmbuf_wseek(&write_buffer, res)) {
LOGGER_ERROR("%s", "wseek error");
break;
} else if (0 > res) {
LOGGER_ERROR("%s", "read error"); // EAGAIN is handled by poller
break;
} else if (0 < res) {
// initial sanitizer
vmbuf_chrcpy(&write_buffer, '\0'); // kill garbage
char *data = ribs_strdup(vmbuf_data(&write_buffer));
//data = strchr(data, '\n') + 1; // skip broken data from initial buffer start. we read from where the file was first observed
ssize_t write_depth = res = strlen(data);
// line doesn't end here
if (data[res - 1] != '\n') {
char *datafringe = ribs_strdup((char *)memrchr(data, '\n', res));
if (SSTRISEMPTY(datafringe))
break;
write_depth = strlen(data) - strlen(datafringe);
*(data + write_depth) = 0;
if (filedef->size != 0) {
char *rebalanced_data = write_file_fringe(fn, data, filedef->fd);
if (NULL != rebalanced_data) {
data = rebalanced_data;
write_depth = strlen(data);
}
}
thashtable_rec_t *rec = thashtable_lookup(delta_push, &filedef->fd, sizeof(filedef->fd));
struct vmbuf kdelta = *(struct vmbuf *)thashtable_get_val(rec);
vmbuf_strcpy(&kdelta, datafringe);
vmbuf_chrcpy(&kdelta, '\0');
}
vmbuf_reset(&write_buffer);
vmbuf_memcpy(&write_buffer, data, write_depth);
vmbuf_chrcpy(&write_buffer, '\0');
write_out_stream(fn, ribs_strdup(vmbuf_data(&write_buffer)));
} else if (0 == res) {
break;
}
}
}
int vmbuf_deflate_ptr2(const void *inbuf, size_t inbuf_size, struct vmbuf *outbuf, int level) {
z_stream strm;
_init_alloc(&strm);
if (Z_OK != deflateInit2(&strm, level, Z_DEFLATED, 15+16, 8, Z_DEFAULT_STRATEGY))
return -1;
int flush, res;
strm.next_in = (void *)inbuf;
strm.avail_in = inbuf_size;
do {
strm.next_out = (uint8_t *)vmbuf_wloc(outbuf);
strm.avail_out = vmbuf_wavail(outbuf);
flush = strm.avail_in == 0 ? Z_FINISH : Z_NO_FLUSH;
if (Z_STREAM_ERROR == (res = deflate(&strm, flush))) {
deflateEnd(&strm);
return -1;
}
vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out);
} while (Z_STREAM_END != res);
deflateEnd(&strm);
return 0;
}
/*
* inline
*/
_RIBS_INLINE_ int http_uri_encode(const char *in, struct vmbuf *out) {
static uint8_t bits[] = {
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x9B, 0x00, 0xFC,
0x01, 0x00, 0x00, 0x78, 0x01, 0x00, 0x00, 0xF8,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
};
static char hex_chars[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F'
};
const uint8_t *inbuf = (const uint8_t *)in;
vmbuf_resize_if_less(out, vmbuf_wlocpos(out)+(strlen(in)*3)+1);
uint8_t *outbuf = (uint8_t *)vmbuf_wloc(out);
if (outbuf == NULL) {
return -1;
}
for ( ; *inbuf; ++inbuf)
{
register uint8_t c = *inbuf;
if (c == ' ')
{
*outbuf++ = '+';
} else if ((bits[c >> 3] & (1 << (c & 7))) > 0)
{
*outbuf++ = '%';
*outbuf++ = hex_chars[c >> 4];
*outbuf++ = hex_chars[c & 15];
}
else
*outbuf++ = c;
}
int vmbuf_inflate_ptr(void *inbuf, size_t inbuf_size, struct vmbuf *outbuf) {
z_stream strm;
_init_alloc(&strm);
if (Z_OK != inflateInit2(&strm, 15+16))
return -1;
strm.next_in = inbuf;
strm.avail_in = inbuf_size;
for (;0 < strm.avail_in;)
{
if (0 > vmbuf_resize_if_less(outbuf, strm.avail_in << 1))
return inflateEnd(&strm), -1;
strm.next_out = (uint8_t *)vmbuf_wloc(outbuf);
strm.avail_out = vmbuf_wavail(outbuf);
int res = inflate(&strm, Z_NO_FLUSH);
vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out);
if (res == Z_STREAM_END)
return inflateEnd(&strm), strm.avail_in == 0 ? 0 : -1; /* return error if inbuf has extra data */
if (Z_OK != res)
return inflateEnd(&strm), -1;
}
inflateEnd(&strm);
return -1; /* if we reached here, we have partial outbuf */
}
void http_client_fiber_main(void) {
struct http_client_context *ctx = (struct http_client_context *)current_ctx->reserved;
int fd = current_ctx->fd;
struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + fd;
TIMEOUT_HANDLER_REMOVE_FD_DATA(fd_data);
int persistent = 0;
epoll_worker_set_last_fd(fd); /* needed in the case where epoll_wait never occured */
/*
* write request
*/
int res;
for (; (res = vmbuf_write(&ctx->request, fd)) == 0; http_client_yield());
if (0 > res) {
LOGGER_PERROR("write");
CLIENT_ERROR();
}
/*
* HTTP header
*/
uint32_t eoh_ofs;
char *data;
char *eoh;
res = vmbuf_read(&ctx->response, fd);
*vmbuf_wloc(&ctx->response) = 0;
READ_MORE_DATA_STR(NULL == (eoh = strstr(data = vmbuf_data(&ctx->response), CRLFCRLF)));
eoh_ofs = eoh - data + SSTRLEN(CRLFCRLF);
*eoh = 0;
char *p = strstr(data, CONNECTION);
if (p != NULL) {
p += SSTRLEN(CONNECTION);
persistent = (0 == SSTRNCMPI(CONNECTION_CLOSE, p) ? 0 : 1);
}
SSTRL(HTTP, "HTTP/");
if (0 != SSTRNCMP(HTTP, data))
CLIENT_ERROR();
p = strchrnul(data, ' ');
int code = (*p ? atoi(p + 1) : 0);
if (0 == code)
CLIENT_ERROR();
do {
if (code == 204 || code == 304) /* No Content, Not Modified */
break;
/*
* content length
*/
char *content_len_str = strstr(data, CONTENT_LENGTH);
if (NULL != content_len_str) {
content_len_str += SSTRLEN(CONTENT_LENGTH);
size_t content_end = eoh_ofs + atoi(content_len_str);
READ_MORE_DATA(vmbuf_wlocpos(&ctx->response) < content_end);
break;
}
/*
* chunked encoding
*/
char *transfer_encoding_str = strstr(data, TRANSFER_ENCODING);
if (NULL != transfer_encoding_str &&
0 == SSTRNCMP(transfer_encoding_str + SSTRLEN(TRANSFER_ENCODING), "chunked")) {
size_t chunk_start = eoh_ofs;
size_t data_start = eoh_ofs;
char *p;
for (;;) {
READ_MORE_DATA_STR(*(p = strchrnul((data = vmbuf_data(&ctx->response)) + chunk_start, '\r')) == 0);
if (0 != SSTRNCMP(CRLF, p))
CLIENT_ERROR();
uint32_t s = strtoul(data + chunk_start, NULL, 16);
if (0 == s) {
vmbuf_wlocset(&ctx->response, data_start);
break;
}
chunk_start = p - data + SSTRLEN(CRLF);
size_t chunk_end = chunk_start + s + SSTRLEN(CRLF);
READ_MORE_DATA(vmbuf_wlocpos(&ctx->response) < chunk_end);
memmove(vmbuf_data(&ctx->response) + data_start, vmbuf_data(&ctx->response) + chunk_start, s);
data_start += s;
chunk_start = chunk_end;
}
break;
}
/*
* older versions of HTTP, terminated by disconnect
*/
for (;; yield()) {
if ((res = vmbuf_read(&ctx->response, fd)) < 0)
CLIENT_ERROR();
if (0 == res)
break; /* remote side closed connection */
}
} while (0);
ctx->content = vmbuf_data_ofs(&ctx->response, eoh_ofs);
ctx->content_length = vmbuf_wlocpos(&ctx->response) - eoh_ofs;
ctx->http_status_code = code;
vmbuf_data_ofs(&ctx->response, eoh_ofs - SSTRLEN(CRLFCRLF))[0] = CR;
*vmbuf_wloc(&ctx->response) = 0;
ctx->persistent = persistent;
if (!persistent)
close(fd);
}
void http_server_fiber_main(void) {
struct http_server_context *ctx = http_server_get_context();
struct http_server *server = ctx->server;
int fd = ctx->fd;
char *URI;
char *headers;
char *content;
size_t content_length;
int res;
ctx->persistent = 0;
vmbuf_init(&ctx->request, server->init_request_size);
vmbuf_init(&ctx->header, server->init_header_size);
vmbuf_init(&ctx->payload, server->init_payload_size);
size_t max_req_size = server->max_req_size;
for (;; http_server_yield()) {
READ_FROM_SOCKET();
if (vmbuf_wlocpos(&ctx->request) > MIN_HTTP_REQ_SIZE)
break;
}
do {
if (0 == SSTRNCMP(GET, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(HEAD, vmbuf_data(&ctx->request))) {
/* GET or HEAD */
while (0 != SSTRNCMP(CRLFCRLF, vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF))) {
http_server_yield();
READ_FROM_SOCKET();
}
/* make sure the string is \0 terminated */
/* this will overwrite the first CR */
*(vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF)) = 0;
char *p = vmbuf_data(&ctx->request);
ctx->persistent = check_persistent(p);
URI = strchrnul(p, ' '); /* can't be NULL GET and HEAD constants have space at the end */
*URI = 0;
++URI; // skip the space
p = strchrnul(URI, '\r'); /* HTTP/1.0 */
headers = p;
if (0 != *headers) /* are headers present? */
headers += SSTRLEN(CRLF); /* skip the new line */
*p = 0;
p = strchrnul(URI, ' '); /* truncate the version part */
*p = 0; /* \0 at the end of URI */
ctx->content = NULL;
ctx->content_len = 0;
/* minimal parsing and call user function */
http_server_process_request(URI, headers);
} else if (0 == SSTRNCMP(POST, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(PUT, vmbuf_data(&ctx->request))) {
/* POST or PUT */
for (;;) {
*vmbuf_wloc(&ctx->request) = 0;
/* wait until we have the header */
if (NULL != (content = strstr(vmbuf_data(&ctx->request), CRLFCRLF)))
break;
http_server_yield();
READ_FROM_SOCKET();
}
*content = 0; /* terminate at the first CR like in GET */
content += SSTRLEN(CRLFCRLF);
size_t content_ofs = content - vmbuf_data(&ctx->request);
if (strstr(vmbuf_data(&ctx->request), EXPECT_100)) {
vmbuf_sprintf(&ctx->header, "%s %s\r\n\r\n", HTTP_SERVER_VER, HTTP_STATUS_100);
if (0 > vmbuf_write(&ctx->header, fd)) {
close(fd);
return;
}
vmbuf_reset(&ctx->header);
}
ctx->persistent = check_persistent(vmbuf_data(&ctx->request));
/* parse the content length */
char *p = strcasestr(vmbuf_data(&ctx->request), CONTENT_LENGTH);
if (NULL == p) {
http_server_response(HTTP_STATUS_411, HTTP_CONTENT_TYPE_TEXT_PLAIN);
break;
}
p += SSTRLEN(CONTENT_LENGTH);
content_length = atoi(p);
for (;;) {
if (content_ofs + content_length <= vmbuf_wlocpos(&ctx->request))
break;
http_server_yield();
READ_FROM_SOCKET();
}
p = vmbuf_data(&ctx->request);
URI = strchrnul(p, ' '); /* can't be NULL PUT and POST constants have space at the end */
*URI = 0;
++URI; /* skip the space */
p = strchrnul(URI, '\r'); /* HTTP/1.0 */
headers = p;
if (0 != *headers) /* are headers present? */
headers += SSTRLEN(CRLF); /* skip the new line */
*p = 0;
p = strchrnul(URI, ' '); /* truncate http version */
*p = 0; /* \0 at the end of URI */
ctx->content = vmbuf_data_ofs(&ctx->request, content_ofs);
*(ctx->content + content_length) = 0;
ctx->content_len = content_length;
/* minimal parsing and call user function */
http_server_process_request(URI, headers);
} else {
http_server_response(HTTP_STATUS_501, HTTP_CONTENT_TYPE_TEXT_PLAIN);
break;
}
} while(0);
if (vmbuf_wlocpos(&ctx->header) > 0) {
epoll_worker_resume_events(fd);
http_server_write();
}
if (ctx->persistent) {
struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + fd;
fd_data->ctx = server->idle_ctx;
timeout_handler_add_fd_data(&server->timeout_handler, fd_data);
} else
close(fd);
}
static inline void end_log_line(int fd) {
*vmbuf_wloc(&log_buf) = '\n';
vmbuf_wseek(&log_buf, 1);
vmbuf_write(&log_buf, fd);
}
const uint8_t *inbuf = (const uint8_t *)in;
vmbuf_resize_if_less(out, vmbuf_wlocpos(out)+(strlen(in)*3)+1);
uint8_t *outbuf = (uint8_t *)vmbuf_wloc(out);
if (outbuf == NULL) {
return -1;
}
for ( ; *inbuf; ++inbuf)
{
register uint8_t c = *inbuf;
if (c == ' ')
{
*outbuf++ = '+';
} else if ((bits[c >> 3] & (1 << (c & 7))) > 0)
{
*outbuf++ = '%';
*outbuf++ = hex_chars[c >> 4];
*outbuf++ = hex_chars[c & 15];
}
else
*outbuf++ = c;
}
vmbuf_wseek(out, outbuf-(uint8_t *)vmbuf_wloc(out));
return 0;
}
_RIBS_INLINE_ int ds_var_field_writer_new_row(struct ds_var_field_writer *dsvfw) {
size_t ofs = file_writer_wlocpos(&dsvfw->data);
*(size_t *)vmbuf_wloc(&dsvfw->ofs_table) = ofs;
return vmbuf_wseek(&dsvfw->ofs_table, sizeof(size_t));
}
static bool
recursive_flush_events (
int inotify_wd,
char **files,
uint32_t num_files) {
struct logz_file_def *filedef = ribs_malloc(num_files * sizeof(struct logz_file_def));
int prev_wd;
size_t evlen = 0;
// size_t evbuf_off = 0;
bool found_unwatchable_dir = false;
bool no_inotify_resources = false;
int inserted = 0;
size_t i;
for (i = 0; i < num_files; i++) {
filedef[i].name = files[i];
size_t fnlen = strlen (filedef[i].name);
if (evlen < fnlen)
evlen = fnlen;
filedef[i].wd = -1;
char *file_fullname = ribs_strdup(filedef[i].name);
char *dir_name = dirname(file_fullname);
size_t dirlen = strlen(dir_name);;
char prev = filedef[i].name[dirlen];
filedef[i].basename_start = basename (file_fullname) - filedef[i].name;
filedef[i].name[dirlen] = '\0';
filedef[i].parent_wd = inotify_add_watch(inotify_wd, dir_name, (IN_CREATE | IN_MOVED_TO));
filedef[i].name[dirlen] = prev;
if(filedef[i].parent_wd < 0) {
if (errno != ENOSPC)
LOGGER_ERROR("cannot watch parent directory of file %s", filedef[i].name);
else {
no_inotify_resources = true;
LOGGER_ERROR("%s", "inotify resources exhausted");
}
found_unwatchable_dir = true;
break;
}
filedef[i].wd = inotify_add_watch(inotify_wd, filedef[i].name, inotify_file_watch_mask);
if (filedef[i].wd < 0) {
if (errno == ENOSPC) {
no_inotify_resources = true;
LOGGER_ERROR("%s", "inotify resources exhausted");
} else if(errno != filedef[i].errnum)
LOGGER_ERROR("cannot watch %s", filedef[i].name);
continue;
}
filedef[i].fd = open(filedef[i].name, O_RDONLY | O_NONBLOCK);
if (0 >= filedef[i].fd) {
LOGGER_ERROR("skipping file %s. cannot open to read", filedef[i].name);
continue;
}
struct stat stats;
if (fstat (filedef[i].fd, &stats) != 0) {
LOGGER_ERROR("skipping file %s.cannot stat", filedef[i].name);
filedef->errnum = errno;
logz_close_fd (filedef[i].fd, filedef[i].name);
filedef->fd = -1;
continue;
}
filedef[i].size = stats.st_size;
lseek (filedef[i].fd, 0, SEEK_END); // no offset enforced
thashtable_insert(tab_event_fds, &filedef[i].wd, sizeof(filedef[i].wd), &filedef[i], sizeof(filedef[i]), &inserted);
struct vmbuf kdelta = VMBUF_INITIALIZER;
vmbuf_init(&kdelta, 4096);
thashtable_insert(delta_push, &filedef[i].fd, sizeof(filedef[i].fd), &kdelta, sizeof(kdelta), &inserted);
}
if(no_inotify_resources || found_unwatchable_dir) {
LOGGER_ERROR("%s", "running low on inotify resources / got an unwatchable directory. Aborting!!");
abort();
}
prev_wd = filedef[num_files -1].wd;
evlen += sizeof (struct inotify_event) + 1;
struct vmbuf evbuf = VMBUF_INITIALIZER;
vmbuf_init(&evbuf, evlen);
ssize_t res = 0;
struct timeval delay; /* how long to wait for file changes. */
delay.tv_sec = (time_t) 0.50;
delay.tv_usec = 1000000 * (0.50 - delay.tv_sec);
fd_set rfd;
FD_ZERO (&rfd);
FD_SET (inotify_wd, &rfd);
while(1) {
if (thashtable_get_size(tab_event_fds) == 0) {
LOGGER_INFO("%s", "no file to read");
return true;
}
{
int file_change = select(inotify_wd + 1, &rfd, NULL, NULL, NULL);
if (file_change == 0)
continue;
else if (file_change == -1) {
LOGGER_ERROR("%s", "error monitoring inotify event");
exit(EXIT_FAILURE);
}
vmbuf_reset(&evbuf);
while (0 < (res = read(inotify_wd, vmbuf_wloc(&evbuf), evlen))) {
if (0 > vmbuf_wseek(&evbuf, res))
return false;
else if (errno == EINTR)
continue;
else if (0 < res)
break;
}
if (errno == EAGAIN && res < 0)
continue;
// res might be 0, or could have overrun memory
if (res == 0) {
LOGGER_ERROR("%s", "error reading inotify event|bad buffer size. aborting to investigate");
abort();
}
// another case when res == 0 or could have overrun memory or got EINVAL
// -- ignored --
// what to do if? .. realloc buffer. but the deal is we're on vmbuf which will grow if world is that bad. we're mostly safe here. hence ignored.
}
struct logz_file_def *tmp;
struct inotify_event *event = (struct inotify_event *)vmbuf_data(&evbuf);
if (event->len) {
// events of lower interest?. these are from watched directory. we'll drop those which we're not watching for and will set watch on those of interest.
size_t x;
for (x = 0; x < num_files; x++) {
if (filedef[x].parent_wd == event->wd
&& strcmp (event->name, filedef[x].name + filedef[x].basename_start))
break;
}
if (x == num_files)
continue;
int wdx = inotify_add_watch (inotify_wd, filedef[x].name, inotify_file_watch_mask);
if (0 > wdx) {
LOGGER_ERROR("cannot watch %s", filedef[x].name);
continue;
}
tmp = &(filedef[x]);
thashtable_remove(tab_event_fds, &tmp->wd, sizeof (tmp->wd));
tmp->wd = wdx;
thashtable_insert(tab_event_fds, &tmp->wd, sizeof(tmp->wd), &tmp, sizeof(struct logz_file_def), &inserted);
// rebalance new found file | make all assertions | we'll read from this as well.
UNUSED(tmp);
} else {
thashtable_rec_t *rec = thashtable_lookup(tab_event_fds, &event->wd, sizeof(event->wd));
tmp = (struct logz_file_def *)thashtable_get_val(rec);
}
if (!tmp) {
continue;
}
if (event->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF)) {
if (event->mask & IN_DELETE_SELF) {
inotify_rm_watch(inotify_wd, tmp->wd);
thashtable_remove(tab_event_fds, &tmp->wd, sizeof(tmp->wd));
}
continue;
}
_flush(tmp, event->wd, &prev_wd);
}
return true;
}
