_RIBS_INLINE_ int read_data(struct timeout_handler *timeout_handler, int cfd, struct vmbuf *response, size_t ofs) { int res = vmbuf_read(response, cfd); if (0 > res) { close(cfd); return -1; } *vmbuf_wloc(response) = 0; char *data, *line; /* Keep reading if * 1.you don't see a CRLF * OR * 2.see a dash and CRLF in the current line */ while ((NULL == strstr((data = vmbuf_data_ofs(response, ofs)), CRLF)) || ((NULL != strchr((line = strrstr(data, CRLF)), '-')) && (NULL != strstr(line, CRLF)))) { if (1 == res) sendemail_yield(timeout_handler, cfd); if (0 >= (res = vmbuf_read(response, cfd))) { LOGGER_PERROR("read"); close(cfd); return -1; } } *vmbuf_wloc(response) = 0; return 0; }
int vmbuf_inflate2(struct vmbuf *inbuf, struct vmbuf *outbuf) { z_stream strm; _init_alloc(&strm); if (Z_OK != inflateInit2(&strm, 15+16)) return -1; for (;;) { strm.next_in = (uint8_t *)vmbuf_rloc(inbuf); strm.avail_in = vmbuf_ravail(inbuf); if (0 == strm.avail_in) break; vmbuf_resize_if_less(outbuf, strm.avail_in << 1); strm.next_out = (uint8_t *)vmbuf_wloc(outbuf); strm.avail_out = vmbuf_wavail(outbuf); int res = inflate(&strm, Z_NO_FLUSH); vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out); if (res == Z_STREAM_END) break; if (Z_OK != res) return inflateEnd(&strm), -1; vmbuf_rseek(inbuf, vmbuf_ravail(inbuf) - strm.avail_in); } inflateEnd(&strm); return 0; }
static void trigger_writer ( const char *filename, struct logz_file_def *filedef) { vmbuf_reset(&write_buffer); ssize_t res; char *fn = basename(ribs_strdup(filename)); while(1) { vmbuf_reset(&write_buffer); res = read(filedef->fd, vmbuf_wloc(&write_buffer), (BUFSIZ + 1024) &~ 1024); filedef->size += res; lseek (filedef->fd, filedef->size, SEEK_SET); if (0 > vmbuf_wseek(&write_buffer, res)) { LOGGER_ERROR("%s", "wseek error"); break; } else if (0 > res) { LOGGER_ERROR("%s", "read error"); // EAGAIN is handled by poller break; } else if (0 < res) { // initial sanitizer vmbuf_chrcpy(&write_buffer, '\0'); // kill garbage char *data = ribs_strdup(vmbuf_data(&write_buffer)); //data = strchr(data, '\n') + 1; // skip broken data from initial buffer start. we read from where the file was first observed ssize_t write_depth = res = strlen(data); // line doesn't end here if (data[res - 1] != '\n') { char *datafringe = ribs_strdup((char *)memrchr(data, '\n', res)); if (SSTRISEMPTY(datafringe)) break; write_depth = strlen(data) - strlen(datafringe); *(data + write_depth) = 0; if (filedef->size != 0) { char *rebalanced_data = write_file_fringe(fn, data, filedef->fd); if (NULL != rebalanced_data) { data = rebalanced_data; write_depth = strlen(data); } } thashtable_rec_t *rec = thashtable_lookup(delta_push, &filedef->fd, sizeof(filedef->fd)); struct vmbuf kdelta = *(struct vmbuf *)thashtable_get_val(rec); vmbuf_strcpy(&kdelta, datafringe); vmbuf_chrcpy(&kdelta, '\0'); } vmbuf_reset(&write_buffer); vmbuf_memcpy(&write_buffer, data, write_depth); vmbuf_chrcpy(&write_buffer, '\0'); write_out_stream(fn, ribs_strdup(vmbuf_data(&write_buffer))); } else if (0 == res) { break; } } }
int vmbuf_deflate_ptr2(const void *inbuf, size_t inbuf_size, struct vmbuf *outbuf, int level) { z_stream strm; _init_alloc(&strm); if (Z_OK != deflateInit2(&strm, level, Z_DEFLATED, 15+16, 8, Z_DEFAULT_STRATEGY)) return -1; int flush, res; strm.next_in = (void *)inbuf; strm.avail_in = inbuf_size; do { strm.next_out = (uint8_t *)vmbuf_wloc(outbuf); strm.avail_out = vmbuf_wavail(outbuf); flush = strm.avail_in == 0 ? Z_FINISH : Z_NO_FLUSH; if (Z_STREAM_ERROR == (res = deflate(&strm, flush))) { deflateEnd(&strm); return -1; } vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out); } while (Z_STREAM_END != res); deflateEnd(&strm); return 0; }
/* * inline */ _RIBS_INLINE_ int http_uri_encode(const char *in, struct vmbuf *out) { static uint8_t bits[] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x9B, 0x00, 0xFC, 0x01, 0x00, 0x00, 0x78, 0x01, 0x00, 0x00, 0xF8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; static char hex_chars[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; const uint8_t *inbuf = (const uint8_t *)in; vmbuf_resize_if_less(out, vmbuf_wlocpos(out)+(strlen(in)*3)+1); uint8_t *outbuf = (uint8_t *)vmbuf_wloc(out); if (outbuf == NULL) { return -1; } for ( ; *inbuf; ++inbuf) { register uint8_t c = *inbuf; if (c == ' ') { *outbuf++ = '+'; } else if ((bits[c >> 3] & (1 << (c & 7))) > 0) { *outbuf++ = '%'; *outbuf++ = hex_chars[c >> 4]; *outbuf++ = hex_chars[c & 15]; } else *outbuf++ = c; }
int vmbuf_inflate_ptr(void *inbuf, size_t inbuf_size, struct vmbuf *outbuf) { z_stream strm; _init_alloc(&strm); if (Z_OK != inflateInit2(&strm, 15+16)) return -1; strm.next_in = inbuf; strm.avail_in = inbuf_size; for (;0 < strm.avail_in;) { if (0 > vmbuf_resize_if_less(outbuf, strm.avail_in << 1)) return inflateEnd(&strm), -1; strm.next_out = (uint8_t *)vmbuf_wloc(outbuf); strm.avail_out = vmbuf_wavail(outbuf); int res = inflate(&strm, Z_NO_FLUSH); vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out); if (res == Z_STREAM_END) return inflateEnd(&strm), strm.avail_in == 0 ? 0 : -1; /* return error if inbuf has extra data */ if (Z_OK != res) return inflateEnd(&strm), -1; } inflateEnd(&strm); return -1; /* if we reached here, we have partial outbuf */ }
void http_client_fiber_main(void) { struct http_client_context *ctx = (struct http_client_context *)current_ctx->reserved; int fd = current_ctx->fd; struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + fd; TIMEOUT_HANDLER_REMOVE_FD_DATA(fd_data); int persistent = 0; epoll_worker_set_last_fd(fd); /* needed in the case where epoll_wait never occured */ /* * write request */ int res; for (; (res = vmbuf_write(&ctx->request, fd)) == 0; http_client_yield()); if (0 > res) { LOGGER_PERROR("write"); CLIENT_ERROR(); } /* * HTTP header */ uint32_t eoh_ofs; char *data; char *eoh; res = vmbuf_read(&ctx->response, fd); *vmbuf_wloc(&ctx->response) = 0; READ_MORE_DATA_STR(NULL == (eoh = strstr(data = vmbuf_data(&ctx->response), CRLFCRLF))); eoh_ofs = eoh - data + SSTRLEN(CRLFCRLF); *eoh = 0; char *p = strstr(data, CONNECTION); if (p != NULL) { p += SSTRLEN(CONNECTION); persistent = (0 == SSTRNCMPI(CONNECTION_CLOSE, p) ? 0 : 1); } SSTRL(HTTP, "HTTP/"); if (0 != SSTRNCMP(HTTP, data)) CLIENT_ERROR(); p = strchrnul(data, ' '); int code = (*p ? atoi(p + 1) : 0); if (0 == code) CLIENT_ERROR(); do { if (code == 204 || code == 304) /* No Content, Not Modified */ break; /* * content length */ char *content_len_str = strstr(data, CONTENT_LENGTH); if (NULL != content_len_str) { content_len_str += SSTRLEN(CONTENT_LENGTH); size_t content_end = eoh_ofs + atoi(content_len_str); READ_MORE_DATA(vmbuf_wlocpos(&ctx->response) < content_end); break; } /* * chunked encoding */ char *transfer_encoding_str = strstr(data, TRANSFER_ENCODING); if (NULL != transfer_encoding_str && 0 == SSTRNCMP(transfer_encoding_str + SSTRLEN(TRANSFER_ENCODING), "chunked")) { size_t chunk_start = eoh_ofs; size_t data_start = eoh_ofs; char *p; for (;;) { READ_MORE_DATA_STR(*(p = strchrnul((data = vmbuf_data(&ctx->response)) + chunk_start, '\r')) == 0); if (0 != SSTRNCMP(CRLF, p)) CLIENT_ERROR(); uint32_t s = strtoul(data + chunk_start, NULL, 16); if (0 == s) { vmbuf_wlocset(&ctx->response, data_start); break; } chunk_start = p - data + SSTRLEN(CRLF); size_t chunk_end = chunk_start + s + SSTRLEN(CRLF); READ_MORE_DATA(vmbuf_wlocpos(&ctx->response) < chunk_end); memmove(vmbuf_data(&ctx->response) + data_start, vmbuf_data(&ctx->response) + chunk_start, s); data_start += s; chunk_start = chunk_end; } break; } /* * older versions of HTTP, terminated by disconnect */ for (;; yield()) { if ((res = vmbuf_read(&ctx->response, fd)) < 0) CLIENT_ERROR(); if (0 == res) break; /* remote side closed connection */ } } while (0); ctx->content = vmbuf_data_ofs(&ctx->response, eoh_ofs); ctx->content_length = vmbuf_wlocpos(&ctx->response) - eoh_ofs; ctx->http_status_code = code; vmbuf_data_ofs(&ctx->response, eoh_ofs - SSTRLEN(CRLFCRLF))[0] = CR; *vmbuf_wloc(&ctx->response) = 0; ctx->persistent = persistent; if (!persistent) close(fd); }
void http_server_fiber_main(void) { struct http_server_context *ctx = http_server_get_context(); struct http_server *server = ctx->server; int fd = ctx->fd; char *URI; char *headers; char *content; size_t content_length; int res; ctx->persistent = 0; vmbuf_init(&ctx->request, server->init_request_size); vmbuf_init(&ctx->header, server->init_header_size); vmbuf_init(&ctx->payload, server->init_payload_size); size_t max_req_size = server->max_req_size; for (;; http_server_yield()) { READ_FROM_SOCKET(); if (vmbuf_wlocpos(&ctx->request) > MIN_HTTP_REQ_SIZE) break; } do { if (0 == SSTRNCMP(GET, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(HEAD, vmbuf_data(&ctx->request))) { /* GET or HEAD */ while (0 != SSTRNCMP(CRLFCRLF, vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF))) { http_server_yield(); READ_FROM_SOCKET(); } /* make sure the string is \0 terminated */ /* this will overwrite the first CR */ *(vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF)) = 0; char *p = vmbuf_data(&ctx->request); ctx->persistent = check_persistent(p); URI = strchrnul(p, ' '); /* can't be NULL GET and HEAD constants have space at the end */ *URI = 0; ++URI; // skip the space p = strchrnul(URI, '\r'); /* HTTP/1.0 */ headers = p; if (0 != *headers) /* are headers present? */ headers += SSTRLEN(CRLF); /* skip the new line */ *p = 0; p = strchrnul(URI, ' '); /* truncate the version part */ *p = 0; /* \0 at the end of URI */ ctx->content = NULL; ctx->content_len = 0; /* minimal parsing and call user function */ http_server_process_request(URI, headers); } else if (0 == SSTRNCMP(POST, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(PUT, vmbuf_data(&ctx->request))) { /* POST or PUT */ for (;;) { *vmbuf_wloc(&ctx->request) = 0; /* wait until we have the header */ if (NULL != (content = strstr(vmbuf_data(&ctx->request), CRLFCRLF))) break; http_server_yield(); READ_FROM_SOCKET(); } *content = 0; /* terminate at the first CR like in GET */ content += SSTRLEN(CRLFCRLF); size_t content_ofs = content - vmbuf_data(&ctx->request); if (strstr(vmbuf_data(&ctx->request), EXPECT_100)) { vmbuf_sprintf(&ctx->header, "%s %s\r\n\r\n", HTTP_SERVER_VER, HTTP_STATUS_100); if (0 > vmbuf_write(&ctx->header, fd)) { close(fd); return; } vmbuf_reset(&ctx->header); } ctx->persistent = check_persistent(vmbuf_data(&ctx->request)); /* parse the content length */ char *p = strcasestr(vmbuf_data(&ctx->request), CONTENT_LENGTH); if (NULL == p) { http_server_response(HTTP_STATUS_411, HTTP_CONTENT_TYPE_TEXT_PLAIN); break; } p += SSTRLEN(CONTENT_LENGTH); content_length = atoi(p); for (;;) { if (content_ofs + content_length <= vmbuf_wlocpos(&ctx->request)) break; http_server_yield(); READ_FROM_SOCKET(); } p = vmbuf_data(&ctx->request); URI = strchrnul(p, ' '); /* can't be NULL PUT and POST constants have space at the end */ *URI = 0; ++URI; /* skip the space */ p = strchrnul(URI, '\r'); /* HTTP/1.0 */ headers = p; if (0 != *headers) /* are headers present? */ headers += SSTRLEN(CRLF); /* skip the new line */ *p = 0; p = strchrnul(URI, ' '); /* truncate http version */ *p = 0; /* \0 at the end of URI */ ctx->content = vmbuf_data_ofs(&ctx->request, content_ofs); *(ctx->content + content_length) = 0; ctx->content_len = content_length; /* minimal parsing and call user function */ http_server_process_request(URI, headers); } else { http_server_response(HTTP_STATUS_501, HTTP_CONTENT_TYPE_TEXT_PLAIN); break; } } while(0); if (vmbuf_wlocpos(&ctx->header) > 0) { epoll_worker_resume_events(fd); http_server_write(); } if (ctx->persistent) { struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + fd; fd_data->ctx = server->idle_ctx; timeout_handler_add_fd_data(&server->timeout_handler, fd_data); } else close(fd); }
static inline void end_log_line(int fd) { *vmbuf_wloc(&log_buf) = '\n'; vmbuf_wseek(&log_buf, 1); vmbuf_write(&log_buf, fd); }
const uint8_t *inbuf = (const uint8_t *)in; vmbuf_resize_if_less(out, vmbuf_wlocpos(out)+(strlen(in)*3)+1); uint8_t *outbuf = (uint8_t *)vmbuf_wloc(out); if (outbuf == NULL) { return -1; } for ( ; *inbuf; ++inbuf) { register uint8_t c = *inbuf; if (c == ' ') { *outbuf++ = '+'; } else if ((bits[c >> 3] & (1 << (c & 7))) > 0) { *outbuf++ = '%'; *outbuf++ = hex_chars[c >> 4]; *outbuf++ = hex_chars[c & 15]; } else *outbuf++ = c; } vmbuf_wseek(out, outbuf-(uint8_t *)vmbuf_wloc(out)); return 0; }
_RIBS_INLINE_ int ds_var_field_writer_new_row(struct ds_var_field_writer *dsvfw) { size_t ofs = file_writer_wlocpos(&dsvfw->data); *(size_t *)vmbuf_wloc(&dsvfw->ofs_table) = ofs; return vmbuf_wseek(&dsvfw->ofs_table, sizeof(size_t)); }
static bool recursive_flush_events ( int inotify_wd, char **files, uint32_t num_files) { struct logz_file_def *filedef = ribs_malloc(num_files * sizeof(struct logz_file_def)); int prev_wd; size_t evlen = 0; // size_t evbuf_off = 0; bool found_unwatchable_dir = false; bool no_inotify_resources = false; int inserted = 0; size_t i; for (i = 0; i < num_files; i++) { filedef[i].name = files[i]; size_t fnlen = strlen (filedef[i].name); if (evlen < fnlen) evlen = fnlen; filedef[i].wd = -1; char *file_fullname = ribs_strdup(filedef[i].name); char *dir_name = dirname(file_fullname); size_t dirlen = strlen(dir_name);; char prev = filedef[i].name[dirlen]; filedef[i].basename_start = basename (file_fullname) - filedef[i].name; filedef[i].name[dirlen] = '\0'; filedef[i].parent_wd = inotify_add_watch(inotify_wd, dir_name, (IN_CREATE | IN_MOVED_TO)); filedef[i].name[dirlen] = prev; if(filedef[i].parent_wd < 0) { if (errno != ENOSPC) LOGGER_ERROR("cannot watch parent directory of file %s", filedef[i].name); else { no_inotify_resources = true; LOGGER_ERROR("%s", "inotify resources exhausted"); } found_unwatchable_dir = true; break; } filedef[i].wd = inotify_add_watch(inotify_wd, filedef[i].name, inotify_file_watch_mask); if (filedef[i].wd < 0) { if (errno == ENOSPC) { no_inotify_resources = true; LOGGER_ERROR("%s", "inotify resources exhausted"); } else if(errno != filedef[i].errnum) LOGGER_ERROR("cannot watch %s", filedef[i].name); continue; } filedef[i].fd = open(filedef[i].name, O_RDONLY | O_NONBLOCK); if (0 >= filedef[i].fd) { LOGGER_ERROR("skipping file %s. cannot open to read", filedef[i].name); continue; } struct stat stats; if (fstat (filedef[i].fd, &stats) != 0) { LOGGER_ERROR("skipping file %s.cannot stat", filedef[i].name); filedef->errnum = errno; logz_close_fd (filedef[i].fd, filedef[i].name); filedef->fd = -1; continue; } filedef[i].size = stats.st_size; lseek (filedef[i].fd, 0, SEEK_END); // no offset enforced thashtable_insert(tab_event_fds, &filedef[i].wd, sizeof(filedef[i].wd), &filedef[i], sizeof(filedef[i]), &inserted); struct vmbuf kdelta = VMBUF_INITIALIZER; vmbuf_init(&kdelta, 4096); thashtable_insert(delta_push, &filedef[i].fd, sizeof(filedef[i].fd), &kdelta, sizeof(kdelta), &inserted); } if(no_inotify_resources || found_unwatchable_dir) { LOGGER_ERROR("%s", "running low on inotify resources / got an unwatchable directory. Aborting!!"); abort(); } prev_wd = filedef[num_files -1].wd; evlen += sizeof (struct inotify_event) + 1; struct vmbuf evbuf = VMBUF_INITIALIZER; vmbuf_init(&evbuf, evlen); ssize_t res = 0; struct timeval delay; /* how long to wait for file changes. */ delay.tv_sec = (time_t) 0.50; delay.tv_usec = 1000000 * (0.50 - delay.tv_sec); fd_set rfd; FD_ZERO (&rfd); FD_SET (inotify_wd, &rfd); while(1) { if (thashtable_get_size(tab_event_fds) == 0) { LOGGER_INFO("%s", "no file to read"); return true; } { int file_change = select(inotify_wd + 1, &rfd, NULL, NULL, NULL); if (file_change == 0) continue; else if (file_change == -1) { LOGGER_ERROR("%s", "error monitoring inotify event"); exit(EXIT_FAILURE); } vmbuf_reset(&evbuf); while (0 < (res = read(inotify_wd, vmbuf_wloc(&evbuf), evlen))) { if (0 > vmbuf_wseek(&evbuf, res)) return false; else if (errno == EINTR) continue; else if (0 < res) break; } if (errno == EAGAIN && res < 0) continue; // res might be 0, or could have overrun memory if (res == 0) { LOGGER_ERROR("%s", "error reading inotify event|bad buffer size. aborting to investigate"); abort(); } // another case when res == 0 or could have overrun memory or got EINVAL // -- ignored -- // what to do if? .. realloc buffer. but the deal is we're on vmbuf which will grow if world is that bad. we're mostly safe here. hence ignored. } struct logz_file_def *tmp; struct inotify_event *event = (struct inotify_event *)vmbuf_data(&evbuf); if (event->len) { // events of lower interest?. these are from watched directory. we'll drop those which we're not watching for and will set watch on those of interest. size_t x; for (x = 0; x < num_files; x++) { if (filedef[x].parent_wd == event->wd && strcmp (event->name, filedef[x].name + filedef[x].basename_start)) break; } if (x == num_files) continue; int wdx = inotify_add_watch (inotify_wd, filedef[x].name, inotify_file_watch_mask); if (0 > wdx) { LOGGER_ERROR("cannot watch %s", filedef[x].name); continue; } tmp = &(filedef[x]); thashtable_remove(tab_event_fds, &tmp->wd, sizeof (tmp->wd)); tmp->wd = wdx; thashtable_insert(tab_event_fds, &tmp->wd, sizeof(tmp->wd), &tmp, sizeof(struct logz_file_def), &inserted); // rebalance new found file | make all assertions | we'll read from this as well. UNUSED(tmp); } else { thashtable_rec_t *rec = thashtable_lookup(tab_event_fds, &event->wd, sizeof(event->wd)); tmp = (struct logz_file_def *)thashtable_get_val(rec); } if (!tmp) { continue; } if (event->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF)) { if (event->mask & IN_DELETE_SELF) { inotify_rm_watch(inotify_wd, tmp->wd); thashtable_remove(tab_event_fds, &tmp->wd, sizeof(tmp->wd)); } continue; } _flush(tmp, event->wd, &prev_wd); } return true; }