int vmbuf_inflate2(struct vmbuf *inbuf, struct vmbuf *outbuf) {
z_stream strm;
_init_alloc(&strm);
if (Z_OK != inflateInit2(&strm, 15+16))
return -1;
for (;;)
{
strm.next_in = (uint8_t *)vmbuf_rloc(inbuf);
strm.avail_in = vmbuf_ravail(inbuf);
if (0 == strm.avail_in)
break;
vmbuf_resize_if_less(outbuf, strm.avail_in << 1);
strm.next_out = (uint8_t *)vmbuf_wloc(outbuf);
strm.avail_out = vmbuf_wavail(outbuf);
int res = inflate(&strm, Z_NO_FLUSH);
vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out);
if (res == Z_STREAM_END)
break;
if (Z_OK != res)
return inflateEnd(&strm), -1;
vmbuf_rseek(inbuf, vmbuf_ravail(inbuf) - strm.avail_in);
}
inflateEnd(&strm);
return 0;
}
static void
trigger_writer (
const char *filename,
struct logz_file_def *filedef) {
vmbuf_reset(&write_buffer);
ssize_t res;
char *fn = basename(ribs_strdup(filename));
while(1) {
vmbuf_reset(&write_buffer);
res = read(filedef->fd, vmbuf_wloc(&write_buffer), (BUFSIZ + 1024) &~ 1024);
filedef->size += res;
lseek (filedef->fd, filedef->size, SEEK_SET);
if (0 > vmbuf_wseek(&write_buffer, res)) {
LOGGER_ERROR("%s", "wseek error");
break;
} else if (0 > res) {
LOGGER_ERROR("%s", "read error"); // EAGAIN is handled by poller
break;
} else if (0 < res) {
// initial sanitizer
vmbuf_chrcpy(&write_buffer, '\0'); // kill garbage
char *data = ribs_strdup(vmbuf_data(&write_buffer));
//data = strchr(data, '\n') + 1; // skip broken data from initial buffer start. we read from where the file was first observed
ssize_t write_depth = res = strlen(data);
// line doesn't end here
if (data[res - 1] != '\n') {
char *datafringe = ribs_strdup((char *)memrchr(data, '\n', res));
if (SSTRISEMPTY(datafringe))
break;
write_depth = strlen(data) - strlen(datafringe);
*(data + write_depth) = 0;
if (filedef->size != 0) {
char *rebalanced_data = write_file_fringe(fn, data, filedef->fd);
if (NULL != rebalanced_data) {
data = rebalanced_data;
write_depth = strlen(data);
}
}
thashtable_rec_t *rec = thashtable_lookup(delta_push, &filedef->fd, sizeof(filedef->fd));
struct vmbuf kdelta = *(struct vmbuf *)thashtable_get_val(rec);
vmbuf_strcpy(&kdelta, datafringe);
vmbuf_chrcpy(&kdelta, '\0');
}
vmbuf_reset(&write_buffer);
vmbuf_memcpy(&write_buffer, data, write_depth);
vmbuf_chrcpy(&write_buffer, '\0');
write_out_stream(fn, ribs_strdup(vmbuf_data(&write_buffer)));
} else if (0 == res) {
break;
}
}
}
int vmbuf_deflate_ptr2(const void *inbuf, size_t inbuf_size, struct vmbuf *outbuf, int level) {
z_stream strm;
_init_alloc(&strm);
if (Z_OK != deflateInit2(&strm, level, Z_DEFLATED, 15+16, 8, Z_DEFAULT_STRATEGY))
return -1;
int flush, res;
strm.next_in = (void *)inbuf;
strm.avail_in = inbuf_size;
do {
strm.next_out = (uint8_t *)vmbuf_wloc(outbuf);
strm.avail_out = vmbuf_wavail(outbuf);
flush = strm.avail_in == 0 ? Z_FINISH : Z_NO_FLUSH;
if (Z_STREAM_ERROR == (res = deflate(&strm, flush))) {
deflateEnd(&strm);
return -1;
}
vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out);
} while (Z_STREAM_END != res);
deflateEnd(&strm);
return 0;
}
int vmbuf_inflate_ptr(void *inbuf, size_t inbuf_size, struct vmbuf *outbuf) {
z_stream strm;
_init_alloc(&strm);
if (Z_OK != inflateInit2(&strm, 15+16))
return -1;
strm.next_in = inbuf;
strm.avail_in = inbuf_size;
for (;0 < strm.avail_in;)
{
if (0 > vmbuf_resize_if_less(outbuf, strm.avail_in << 1))
return inflateEnd(&strm), -1;
strm.next_out = (uint8_t *)vmbuf_wloc(outbuf);
strm.avail_out = vmbuf_wavail(outbuf);
int res = inflate(&strm, Z_NO_FLUSH);
vmbuf_wseek(outbuf, vmbuf_wavail(outbuf) - strm.avail_out);
if (res == Z_STREAM_END)
return inflateEnd(&strm), strm.avail_in == 0 ? 0 : -1; /* return error if inbuf has extra data */
if (Z_OK != res)
return inflateEnd(&strm), -1;
}
inflateEnd(&strm);
return -1; /* if we reached here, we have partial outbuf */
}
static inline void end_log_line(int fd) {
*vmbuf_wloc(&log_buf) = '\n';
vmbuf_wseek(&log_buf, 1);
vmbuf_write(&log_buf, fd);
}
_RIBS_INLINE_ int ds_var_field_writer_new_row(struct ds_var_field_writer *dsvfw) {
size_t ofs = file_writer_wlocpos(&dsvfw->data);
*(size_t *)vmbuf_wloc(&dsvfw->ofs_table) = ofs;
return vmbuf_wseek(&dsvfw->ofs_table, sizeof(size_t));
}
static bool
recursive_flush_events (
int inotify_wd,
char **files,
uint32_t num_files) {
struct logz_file_def *filedef = ribs_malloc(num_files * sizeof(struct logz_file_def));
int prev_wd;
size_t evlen = 0;
// size_t evbuf_off = 0;
bool found_unwatchable_dir = false;
bool no_inotify_resources = false;
int inserted = 0;
size_t i;
for (i = 0; i < num_files; i++) {
filedef[i].name = files[i];
size_t fnlen = strlen (filedef[i].name);
if (evlen < fnlen)
evlen = fnlen;
filedef[i].wd = -1;
char *file_fullname = ribs_strdup(filedef[i].name);
char *dir_name = dirname(file_fullname);
size_t dirlen = strlen(dir_name);;
char prev = filedef[i].name[dirlen];
filedef[i].basename_start = basename (file_fullname) - filedef[i].name;
filedef[i].name[dirlen] = '\0';
filedef[i].parent_wd = inotify_add_watch(inotify_wd, dir_name, (IN_CREATE | IN_MOVED_TO));
filedef[i].name[dirlen] = prev;
if(filedef[i].parent_wd < 0) {
if (errno != ENOSPC)
LOGGER_ERROR("cannot watch parent directory of file %s", filedef[i].name);
else {
no_inotify_resources = true;
LOGGER_ERROR("%s", "inotify resources exhausted");
}
found_unwatchable_dir = true;
break;
}
filedef[i].wd = inotify_add_watch(inotify_wd, filedef[i].name, inotify_file_watch_mask);
if (filedef[i].wd < 0) {
if (errno == ENOSPC) {
no_inotify_resources = true;
LOGGER_ERROR("%s", "inotify resources exhausted");
} else if(errno != filedef[i].errnum)
LOGGER_ERROR("cannot watch %s", filedef[i].name);
continue;
}
filedef[i].fd = open(filedef[i].name, O_RDONLY | O_NONBLOCK);
if (0 >= filedef[i].fd) {
LOGGER_ERROR("skipping file %s. cannot open to read", filedef[i].name);
continue;
}
struct stat stats;
if (fstat (filedef[i].fd, &stats) != 0) {
LOGGER_ERROR("skipping file %s.cannot stat", filedef[i].name);
filedef->errnum = errno;
logz_close_fd (filedef[i].fd, filedef[i].name);
filedef->fd = -1;
continue;
}
filedef[i].size = stats.st_size;
lseek (filedef[i].fd, 0, SEEK_END); // no offset enforced
thashtable_insert(tab_event_fds, &filedef[i].wd, sizeof(filedef[i].wd), &filedef[i], sizeof(filedef[i]), &inserted);
struct vmbuf kdelta = VMBUF_INITIALIZER;
vmbuf_init(&kdelta, 4096);
thashtable_insert(delta_push, &filedef[i].fd, sizeof(filedef[i].fd), &kdelta, sizeof(kdelta), &inserted);
}
if(no_inotify_resources || found_unwatchable_dir) {
LOGGER_ERROR("%s", "running low on inotify resources / got an unwatchable directory. Aborting!!");
abort();
}
prev_wd = filedef[num_files -1].wd;
evlen += sizeof (struct inotify_event) + 1;
struct vmbuf evbuf = VMBUF_INITIALIZER;
vmbuf_init(&evbuf, evlen);
ssize_t res = 0;
struct timeval delay; /* how long to wait for file changes. */
delay.tv_sec = (time_t) 0.50;
delay.tv_usec = 1000000 * (0.50 - delay.tv_sec);
fd_set rfd;
FD_ZERO (&rfd);
FD_SET (inotify_wd, &rfd);
while(1) {
if (thashtable_get_size(tab_event_fds) == 0) {
LOGGER_INFO("%s", "no file to read");
return true;
}
{
int file_change = select(inotify_wd + 1, &rfd, NULL, NULL, NULL);
if (file_change == 0)
continue;
else if (file_change == -1) {
LOGGER_ERROR("%s", "error monitoring inotify event");
exit(EXIT_FAILURE);
}
vmbuf_reset(&evbuf);
while (0 < (res = read(inotify_wd, vmbuf_wloc(&evbuf), evlen))) {
if (0 > vmbuf_wseek(&evbuf, res))
return false;
else if (errno == EINTR)
continue;
else if (0 < res)
break;
}
if (errno == EAGAIN && res < 0)
continue;
// res might be 0, or could have overrun memory
if (res == 0) {
LOGGER_ERROR("%s", "error reading inotify event|bad buffer size. aborting to investigate");
abort();
}
// another case when res == 0 or could have overrun memory or got EINVAL
// -- ignored --
// what to do if? .. realloc buffer. but the deal is we're on vmbuf which will grow if world is that bad. we're mostly safe here. hence ignored.
}
struct logz_file_def *tmp;
struct inotify_event *event = (struct inotify_event *)vmbuf_data(&evbuf);
if (event->len) {
// events of lower interest?. these are from watched directory. we'll drop those which we're not watching for and will set watch on those of interest.
size_t x;
for (x = 0; x < num_files; x++) {
if (filedef[x].parent_wd == event->wd
&& strcmp (event->name, filedef[x].name + filedef[x].basename_start))
break;
}
if (x == num_files)
continue;
int wdx = inotify_add_watch (inotify_wd, filedef[x].name, inotify_file_watch_mask);
if (0 > wdx) {
LOGGER_ERROR("cannot watch %s", filedef[x].name);
continue;
}
tmp = &(filedef[x]);
thashtable_remove(tab_event_fds, &tmp->wd, sizeof (tmp->wd));
tmp->wd = wdx;
thashtable_insert(tab_event_fds, &tmp->wd, sizeof(tmp->wd), &tmp, sizeof(struct logz_file_def), &inserted);
// rebalance new found file | make all assertions | we'll read from this as well.
UNUSED(tmp);
} else {
thashtable_rec_t *rec = thashtable_lookup(tab_event_fds, &event->wd, sizeof(event->wd));
tmp = (struct logz_file_def *)thashtable_get_val(rec);
}
if (!tmp) {
continue;
}
if (event->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF)) {
if (event->mask & IN_DELETE_SELF) {
inotify_rm_watch(inotify_wd, tmp->wd);
thashtable_remove(tab_event_fds, &tmp->wd, sizeof(tmp->wd));
}
continue;
}
_flush(tmp, event->wd, &prev_wd);
}
return true;
}
