static void
trigger_writer (
const char *filename,
struct logz_file_def *filedef) {
vmbuf_reset(&write_buffer);
ssize_t res;
char *fn = basename(ribs_strdup(filename));
while(1) {
vmbuf_reset(&write_buffer);
res = read(filedef->fd, vmbuf_wloc(&write_buffer), (BUFSIZ + 1024) &~ 1024);
filedef->size += res;
lseek (filedef->fd, filedef->size, SEEK_SET);
if (0 > vmbuf_wseek(&write_buffer, res)) {
LOGGER_ERROR("%s", "wseek error");
break;
} else if (0 > res) {
LOGGER_ERROR("%s", "read error"); // EAGAIN is handled by poller
break;
} else if (0 < res) {
// initial sanitizer
vmbuf_chrcpy(&write_buffer, '\0'); // kill garbage
char *data = ribs_strdup(vmbuf_data(&write_buffer));
//data = strchr(data, '\n') + 1; // skip broken data from initial buffer start. we read from where the file was first observed
ssize_t write_depth = res = strlen(data);
// line doesn't end here
if (data[res - 1] != '\n') {
char *datafringe = ribs_strdup((char *)memrchr(data, '\n', res));
if (SSTRISEMPTY(datafringe))
break;
write_depth = strlen(data) - strlen(datafringe);
*(data + write_depth) = 0;
if (filedef->size != 0) {
char *rebalanced_data = write_file_fringe(fn, data, filedef->fd);
if (NULL != rebalanced_data) {
data = rebalanced_data;
write_depth = strlen(data);
}
}
thashtable_rec_t *rec = thashtable_lookup(delta_push, &filedef->fd, sizeof(filedef->fd));
struct vmbuf kdelta = *(struct vmbuf *)thashtable_get_val(rec);
vmbuf_strcpy(&kdelta, datafringe);
vmbuf_chrcpy(&kdelta, '\0');
}
vmbuf_reset(&write_buffer);
vmbuf_memcpy(&write_buffer, data, write_depth);
vmbuf_chrcpy(&write_buffer, '\0');
write_out_stream(fn, ribs_strdup(vmbuf_data(&write_buffer)));
} else if (0 == res) {
break;
}
}
}
void http_server_response_sprintf(const char *status, const char *content_type, const char *format, ...) {
struct http_server_context *ctx = http_server_get_context();
vmbuf_reset(&ctx->header);
vmbuf_reset(&ctx->payload);
http_server_header_start(status, content_type);
va_list ap;
va_start(ap, format);
vmbuf_vsprintf(&ctx->payload, format, ap);
va_end(ap);
http_server_header_content_length();
http_server_header_close();
}
int http_server_sendfile2(const char *filename, const char *additional_headers, const char *ext) {
if (0 == *filename)
filename = ".";
struct http_server_context *ctx = http_server_get_context();
int ffd = open(filename, O_RDONLY);
if (ffd < 0)
return HTTP_SERVER_NOT_FOUND;
struct stat st;
if (0 > fstat(ffd, &st)) {
LOGGER_PERROR(filename);
close(ffd);
return HTTP_SERVER_NOT_FOUND;
}
if (S_ISDIR(st.st_mode)) {
close(ffd);
return 1;
}
vmbuf_reset(&ctx->header);
if (NULL != ext)
http_server_header_start(HTTP_STATUS_200, mime_types_by_ext(ext));
else
http_server_header_start(HTTP_STATUS_200, mime_types_by_filename(filename));
vmbuf_sprintf(&ctx->header, "%s%lu", CONTENT_LENGTH, st.st_size);
if (additional_headers)
vmbuf_strcpy(&ctx->header, additional_headers);
http_server_header_close();
int res = http_server_sendfile_payload(ffd, st.st_size);
close(ffd);
if (0 > res)
LOGGER_PERROR(filename);
return res;
}
void http_server_response(const char *status, const char *content_type) {
struct http_server_context *ctx = http_server_get_context();
vmbuf_reset(&ctx->header);
http_server_header_start(status, content_type);
http_server_header_content_length();
http_server_header_close();
}
static inline int make_ht_key(struct vmbuf *buf, const struct mysql_login_info *info) {
vmbuf_reset(buf);
if (0 > vmbuf_sprintf(buf, "%s,%s,%s,%s,%u",
info->host, info->user, info->pass, info->db, info->port))
return -1;
return 0;
}
static void
write_out_stream (const char *filename, char *data) {
vmbuf_reset(&write_buffer);
vmbuf_sprintf(&write_buffer, "{ \"message\": \"%s|%s|", hostname, filename);
json_escape_str_vmb(&write_buffer, data);
vmbuf_strcpy(&write_buffer, "\" }");
vmbuf_chrcpy(&write_buffer, '\0');
if (write_to_file) {
if (0 > file_writer_write(&fw, vmbuf_data(&write_buffer), vmbuf_wlocpos(&write_buffer))) {
LOGGER_ERROR("%s", "failed write attempt on outfile| aborting to diagnose!");
abort();
}
return;
}
int threshold = INTERFACE_ONERROR_RETRY_THRESHOLD;
while (0 > post_to_interface(vmbuf_data(&write_buffer), vmbuf_wlocpos(&write_buffer)) && (0 < threshold--)) {
if (0 == post_to_interface(vmbuf_data(&write_buffer), vmbuf_wlocpos(&write_buffer))) {
LOGGER_ERROR("post failed to %s, issuing reattempt#%d", eserv.hostname, threshold);
--failure;
break;
}
}
}
static char *
write_file_fringe (const char *filename, char *data, int fd) {
thashtable_rec_t *rec = thashtable_lookup(delta_push, &fd, sizeof(fd));
struct vmbuf delta = *(struct vmbuf *)thashtable_get_val(rec);
char *past = vmbuf_data(&delta);
char *lookahead = NULL;
if (!SSTRISEMPTY(past)) {
lookahead = strchr(data, '\n');
char *trailing_past = ribs_malloc_sprintf("%.*s", ((int)strlen(data) - (int)strlen(lookahead)), data);
if (trailing_past) {
char *d_composite = ribs_malloc_sprintf("%s%s", past, trailing_past);
write_out_stream(filename, d_composite);
d_composite += strlen(d_composite); // advance by that
vmbuf_reset(&delta);
}
}
return lookahead;
}
int http_server_sendfile_payload(int ffd, off_t size) {
struct http_server_context *ctx = http_server_get_context();
int fd = ctx->fd;
int option = 1;
if (0 > setsockopt(fd, IPPROTO_TCP, TCP_CORK, &option, sizeof(option)))
LOGGER_PERROR("TCP_CORK set");
epoll_worker_resume_events(ctx->fd);
http_server_write();
vmbuf_reset(&ctx->header);
off_t ofs = 0;
for (;;http_server_yield()) {
if (0 > sendfile(fd, ffd, &ofs, size - ofs) && EAGAIN != errno)
return ctx->persistent = 0, -1;
if (ofs >= size) break;
}
option = 0;
if (0 > setsockopt(fd, IPPROTO_TCP, TCP_CORK, &option, sizeof(option)))
LOGGER_PERROR("TCP_CORK release");
return 0;
}
static int
http_client_pool_post_request2(
struct http_client_pool *http_client_pool,
struct in_addr addr, uint16_t port, const char *hostname,
const char *data, size_t size_of_data, const char *format, ...) {
struct http_client_context *cctx = http_client_pool_create_client2(http_client_pool, addr, port, hostname, NULL);
if (NULL == cctx)
return -1;
vmbuf_reset(&cctx->request);
vmbuf_strcpy(&cctx->request, "POST ");
va_list ap;
va_start(ap, format);
vmbuf_vsprintf(&cctx->request, format, ap);
va_end(ap);
vmbuf_sprintf(&cctx->request, " HTTP/1.1\r\nHost: %s\r\nContent-Type: application/json\r\nContent-Length: %zu\r\n\r\n", hostname, size_of_data);
vmbuf_memcpy(&cctx->request, data, size_of_data);
vmbuf_chrcpy(&cctx->request, '\0');
if (0 > http_client_send_request(cctx))
return http_client_free(cctx), -1;
return 0;
}
/*
* Executes the given SQL query after binding the parameters as indicated
* by 'params'. The results are bound to the provided pointers with order
* and types described by 'fields'. Number of var args should match the
* combined length of 'params' and 'fields'. The number of 'params' must
* be exactly the same as the number of ?s in the query.
*
* Valid format specifies:
* d: MYSQL_TYPE_LONG (signed)
* D: MYSQL_TYPE_LONG (unsigned)
* f: MYSQL_TYPE_DOUBLE
* s: MYSQL_TYPE_STRING
* S: MYSQL_TYPE_STRING, buffer size followed by pointer to the data
*
* note that strings are (char *) for input parameters and (char **) for output parameters
*
* Examples:
* int id;
* char *name, *new_name, *status;
*
* SSTR(query, "SELECT name FROM table WHERE id = ?");
* mysql_helper_stmt(&mh, query, SSTRLEN(query), "d", "s", &id, &name);
*
* SSTR(query, "UPDATE table SET name = ? WHERE id = ?");
* mysql_helper_stmt(&mh, query, SSTRLEN(query), "sd", "", new_name, &id);
*
* SSTR(query, "INSERT INTO table (status, name) VALUES (?, ?)");
* mysql_helper_stmt(&mh, query, SSTRLEN(query), "ss", "", status, name);
*/
int mysql_helper_stmt(struct mysql_helper *mysql_helper,
const char *query,
size_t query_len,
const char *params,
const char *fields,
...)
{
/*
* 0. Cleanup from previous call
*/
if (mysql_helper->stmt) {
mysql_stmt_close(mysql_helper->stmt);
mysql_helper->stmt = NULL;
vmbuf_reset(&mysql_helper->buf);
}
/*
* 1. Prepare statement
*/
mysql_helper->stmt = mysql_stmt_init(&mysql_helper->mysql);
if (!mysql_helper->stmt)
return report_error(mysql_helper);
if (0 != mysql_stmt_prepare(mysql_helper->stmt, query, query_len))
return report_stmt_error(mysql_helper);
/*
* 2. Bind parameters if there are any
*/
uint32_t nparams = strlen(params);
uint32_t n = mysql_stmt_param_count(mysql_helper->stmt);
if (nparams != n) {
LOGGER_ERROR("num params != num ?s in query (%u != %u)", nparams, n);
return -1;
}
const char *p;
uint32_t i;
va_list ap;
va_start(ap, fields);
/* TODO: move to internal vmbuf (mysql_helper), so
mysql_stmt_execute can be called multiple times when inserting
data */
unsigned long plengths[nparams];
int ptypes[nparams];
my_bool pnulls[nparams];
MYSQL_BIND pbind[nparams];
if (nparams > 0) {
memset(pbind, 0, sizeof(pbind));
MYSQL_BIND *pbind_ptr = pbind;
for (i = 0, p = params; *p; ++p, ++pbind_ptr, ++i) {
char c = *p;
char *str;
switch(tolower(c)) {
case 'd':
ptypes[i] = MYSQL_TYPE_LONG;
pbind_ptr->buffer = va_arg(ap, int *);
pnulls[i] = (pbind_ptr->buffer == NULL);
pbind_ptr->is_unsigned = isupper(c) ? 1 : 0;
break;
case 'f':
ptypes[i] = MYSQL_TYPE_DOUBLE;
pbind_ptr->buffer = va_arg(ap, double *);
pnulls[i] = (pbind_ptr->buffer == NULL);
break;
case 's':
ptypes[i] = MYSQL_TYPE_STRING;
if (isupper(c)) {
plengths[i] = va_arg(ap, size_t);
str = va_arg(ap, char *);
} else {
str = va_arg(ap, char *);
plengths[i] = strlen(str);
}
pnulls[i] = (str == NULL);
pbind_ptr->buffer = str;
pbind_ptr->buffer_length = plengths[i];
pbind_ptr->length = &plengths[i];
break;
}
pbind_ptr->buffer_type = ptypes[i];
pbind_ptr->is_null = &pnulls[i];
}
void http_server_fiber_main(void) {
struct http_server_context *ctx = http_server_get_context();
struct http_server *server = ctx->server;
int fd = ctx->fd;
char *URI;
char *headers;
char *content;
size_t content_length;
int res;
ctx->persistent = 0;
vmbuf_init(&ctx->request, server->init_request_size);
vmbuf_init(&ctx->header, server->init_header_size);
vmbuf_init(&ctx->payload, server->init_payload_size);
size_t max_req_size = server->max_req_size;
for (;; http_server_yield()) {
READ_FROM_SOCKET();
if (vmbuf_wlocpos(&ctx->request) > MIN_HTTP_REQ_SIZE)
break;
}
do {
if (0 == SSTRNCMP(GET, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(HEAD, vmbuf_data(&ctx->request))) {
/* GET or HEAD */
while (0 != SSTRNCMP(CRLFCRLF, vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF))) {
http_server_yield();
READ_FROM_SOCKET();
}
/* make sure the string is \0 terminated */
/* this will overwrite the first CR */
*(vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF)) = 0;
char *p = vmbuf_data(&ctx->request);
ctx->persistent = check_persistent(p);
URI = strchrnul(p, ' '); /* can't be NULL GET and HEAD constants have space at the end */
*URI = 0;
++URI; // skip the space
p = strchrnul(URI, '\r'); /* HTTP/1.0 */
headers = p;
if (0 != *headers) /* are headers present? */
headers += SSTRLEN(CRLF); /* skip the new line */
*p = 0;
p = strchrnul(URI, ' '); /* truncate the version part */
*p = 0; /* \0 at the end of URI */
ctx->content = NULL;
ctx->content_len = 0;
/* minimal parsing and call user function */
http_server_process_request(URI, headers);
} else if (0 == SSTRNCMP(POST, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(PUT, vmbuf_data(&ctx->request))) {
/* POST or PUT */
for (;;) {
*vmbuf_wloc(&ctx->request) = 0;
/* wait until we have the header */
if (NULL != (content = strstr(vmbuf_data(&ctx->request), CRLFCRLF)))
break;
http_server_yield();
READ_FROM_SOCKET();
}
*content = 0; /* terminate at the first CR like in GET */
content += SSTRLEN(CRLFCRLF);
size_t content_ofs = content - vmbuf_data(&ctx->request);
if (strstr(vmbuf_data(&ctx->request), EXPECT_100)) {
vmbuf_sprintf(&ctx->header, "%s %s\r\n\r\n", HTTP_SERVER_VER, HTTP_STATUS_100);
if (0 > vmbuf_write(&ctx->header, fd)) {
close(fd);
return;
}
vmbuf_reset(&ctx->header);
}
ctx->persistent = check_persistent(vmbuf_data(&ctx->request));
/* parse the content length */
char *p = strcasestr(vmbuf_data(&ctx->request), CONTENT_LENGTH);
if (NULL == p) {
http_server_response(HTTP_STATUS_411, HTTP_CONTENT_TYPE_TEXT_PLAIN);
break;
}
p += SSTRLEN(CONTENT_LENGTH);
content_length = atoi(p);
for (;;) {
if (content_ofs + content_length <= vmbuf_wlocpos(&ctx->request))
break;
http_server_yield();
READ_FROM_SOCKET();
}
p = vmbuf_data(&ctx->request);
URI = strchrnul(p, ' '); /* can't be NULL PUT and POST constants have space at the end */
*URI = 0;
++URI; /* skip the space */
p = strchrnul(URI, '\r'); /* HTTP/1.0 */
headers = p;
if (0 != *headers) /* are headers present? */
headers += SSTRLEN(CRLF); /* skip the new line */
*p = 0;
p = strchrnul(URI, ' '); /* truncate http version */
*p = 0; /* \0 at the end of URI */
ctx->content = vmbuf_data_ofs(&ctx->request, content_ofs);
*(ctx->content + content_length) = 0;
ctx->content_len = content_length;
/* minimal parsing and call user function */
http_server_process_request(URI, headers);
} else {
http_server_response(HTTP_STATUS_501, HTTP_CONTENT_TYPE_TEXT_PLAIN);
break;
}
} while(0);
if (vmbuf_wlocpos(&ctx->header) > 0) {
epoll_worker_resume_events(fd);
http_server_write();
}
if (ctx->persistent) {
struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + fd;
fd_data->ctx = server->idle_ctx;
timeout_handler_add_fd_data(&server->timeout_handler, fd_data);
} else
close(fd);
}
int mysql_dumper_dump(struct mysql_login_info *mysql_login_info, const char *outputdir, const char *dbname, const char *tablename, const char *query, size_t query_len, struct mysql_dumper_type *types) {
MYSQL mysql;
MYSQL_STMT *stmt = NULL;
mysql_init(&mysql);
my_bool b_flag = 1;
if (0 != mysql_options(&mysql, MYSQL_OPT_RECONNECT, (const char *)&b_flag))
return report_error(&mysql);
if (NULL == mysql_real_connect(&mysql, mysql_login_info->host, mysql_login_info->user, mysql_login_info->pass, mysql_login_info->db, mysql_login_info->port, NULL, CLIENT_COMPRESS))
return report_error(&mysql);
b_flag = 0;
if (0 != mysql_options(&mysql, MYSQL_REPORT_DATA_TRUNCATION, (const char *)&b_flag))
return report_error(&mysql);
stmt = mysql_stmt_init(&mysql);
if (!stmt)
return report_error(&mysql);
if (0 != mysql_stmt_prepare(stmt, query, query_len))
return report_stmt_error(&mysql, stmt);
MYSQL_RES *rs = mysql_stmt_result_metadata(stmt);
if (!rs)
return report_stmt_error(&mysql, stmt);
unsigned int n = mysql_num_fields(rs);
MYSQL_FIELD *fields = mysql_fetch_fields(rs);
int field_types[n];
MYSQL_BIND bind[n];
my_bool is_null[n];
unsigned long length[n];
my_bool error[n];
memset(bind, 0, sizeof(MYSQL_BIND) * n);
int null_terminate_str[n];
memset(null_terminate_str, 0, sizeof(int) * n);
struct file_writer ffields[n];
struct file_writer vfields[2][n];
struct vmbuf buf = VMBUF_INITIALIZER;
vmbuf_init(&buf, 4096);
vmbuf_sprintf(&buf, "%s/%s/%s/schema.txt", outputdir, dbname, tablename);
mkdir_for_file_recursive(vmbuf_data(&buf));
int fdschema = creat(vmbuf_data(&buf), 0644);
struct vmfile ds_txt = VMFILE_INITIALIZER;
vmbuf_reset(&buf);
vmbuf_sprintf(&buf, "%s/%s/%s/ds.txt", outputdir, dbname, tablename);
if (0 > vmfile_init(&ds_txt, vmbuf_data(&buf), 4096))
return LOGGER_ERROR("failed to create: %s", vmbuf_data(&buf)), vmbuf_free(&buf), -1;
vmfile_sprintf(&ds_txt, "DS_LOADER_BEGIN()\n");
vmfile_sprintf(&ds_txt, "/*\n * DB: %s\n */\n", dbname);
vmfile_sprintf(&ds_txt, "#undef DB_NAME\n#define DB_NAME %s\n", dbname);
ssize_t len = 80 - strlen(tablename);
if (len < 0) len = 4;
char header[len];
memset(header, '=', len);
vmfile_sprintf(&ds_txt, "/* %.*s[ %s ]%.*s */\n", (int)len / 2, header, tablename, (int)(len - (len / 2)), header);
vmfile_sprintf(&ds_txt, "# undef TABLE_NAME\n# define TABLE_NAME %s\n", tablename);
/*
* initialize output files
*/
unsigned int i;
for (i = 0; i < n; ++i) {
file_writer_make(&ffields[i]);
file_writer_make(&vfields[0][i]);
file_writer_make(&vfields[1][i]);
}
struct hashtable ht_types = HASHTABLE_INITIALIZER;
hashtable_init(&ht_types, 32);
if (NULL != types) {
struct mysql_dumper_type *t = types;
for (; t->name; ++t) {
/* storing ptr here which points to static str */
hashtable_insert(&ht_types, t->name, strlen(t->name), t, sizeof(struct mysql_dumper_type));
}
}
/*
* parse meta data and construct bind array
*/
int err = 0;
for (i = 0; i < n; ++i) {
field_types[i] = fields[i].type;
bind[i].is_unsigned = IS_UNSIGNED(fields[i].flags);
int64_t ds_type = -1;
const char *ds_type_str = "VAR";
/*
* handle overrides
*/
while (NULL != types) {
uint32_t ofs = hashtable_lookup(&ht_types, fields[i].name, strlen(fields[i].name));
if (!ofs) break;
struct mysql_dumper_type *type = (struct mysql_dumper_type *)hashtable_get_val(&ht_types, ofs);
null_terminate_str[i] = MYSQL_DUMPER_CSTR & type->flags;
if (type->mysql_type)
field_types[i] = type->mysql_type;
bind[i].is_unsigned = (type->flags & MYSQL_DUMPER_UNSIGNED) > 0 ? 1 : 0;
break;
}
vmbuf_reset(&buf);
vmbuf_sprintf(&buf, "%s/%s/%s/%s", outputdir, dbname, tablename, fields[i].name);
mkdir_for_file_recursive(vmbuf_data(&buf));
if (is_var_length_field(field_types[i])) {
size_t ofs = vmbuf_wlocpos(&buf);
vmbuf_sprintf(&buf, ".ofs");
if (0 > (err = file_writer_init(&vfields[0][i], vmbuf_data(&buf))))
break;
vmbuf_wlocset(&buf, ofs);
vmbuf_sprintf(&buf, ".dat");
if (0 > (err = file_writer_init(&vfields[1][i], vmbuf_data(&buf))))
break;
} else {
ds_type = get_ds_type(field_types[i], bind[i].is_unsigned);
const char *s = get_ds_type_str(ds_type);
if (*s) ds_type_str = s;
if (0 > (err = file_writer_init(&ffields[i], vmbuf_data(&buf))) ||
0 > (err = file_writer_write(&ffields[i], &ds_type, sizeof(ds_type))))
break;;
}
len = ribs_mysql_get_storage_size(field_types[i], fields[i].length);
if (fdschema > 0)
dprintf(fdschema, "%03d name = %s, size=%zu, length=%lu, type=%s (%s), is_prikey=%d, ds_type=%s\n", i, fields[i].name, len, fields[i].length, ribs_mysql_get_type_name(field_types[i]), bind[i].is_unsigned ? "unsigned" : "signed", IS_PRI_KEY(fields[i].flags), ds_type_str);
if (is_var_length_field(field_types[i])) {
vmfile_sprintf(&ds_txt, " DS_VAR_FIELD_LOADER(%s)\n", fields[i].name);
} else {
vmfile_sprintf(&ds_txt, " DS_FIELD_LOADER(%s, %s)\n", ds_type_str, fields[i].name);
}
bind[i].buffer_type = field_types[i];
bind[i].buffer_length = len;
bind[i].buffer = malloc(len);
bind[i].is_null = &is_null[i];
bind[i].length = &length[i];
bind[i].error = &error[i];
}
hashtable_free(&ht_types);
mysql_free_result(rs);
close(fdschema);
//vmfile_sprintf(&ds_txt, "/*\n * TABLE END: %s\n */\n", tablename);
vmfile_sprintf(&ds_txt, "DS_LOADER_END()\n");
vmfile_close(&ds_txt);
/*
* execute & bind
*/
if (0 != err ||
0 != mysql_stmt_execute(stmt) ||
0 != mysql_stmt_bind_result(stmt, bind)) {
err = -1;
report_stmt_error(&mysql, stmt);
goto dumper_close_writer;
}
char zeros[4096];
memset(zeros, 0, sizeof(zeros));
int mysql_err = 0;
size_t count = 0, num_rows_errors = 0;
/*
* write all rows to output files
*/
while (0 == (mysql_err = mysql_stmt_fetch(stmt))) {
int b = 0;
for (i = 0; i < n && !b; ++i)
b = b || error[i];
if (b) {
++num_rows_errors;
continue;
}
for (i = 0; i < n; ++i) {
if (is_var_length_field(field_types[i])) {
size_t ofs = file_writer_wlocpos(&vfields[1][i]);
if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs))) ||
0 > (err = file_writer_write(&vfields[1][i], is_null[i] ? NULL : bind[i].buffer, is_null[i] ? 0 : length[i])))
goto dumper_error;
if (null_terminate_str[i]) {
const char c = '\0';
if (0 > (err = file_writer_write(&vfields[1][i], &c, sizeof(c))))
goto dumper_error;
}
} else {
if (0 > (err = file_writer_write(&ffields[i], is_null[i] ? zeros : bind[i].buffer, bind[i].buffer_length)))
goto dumper_error;
}
}
++count;
}
/* no dumper errors */
goto dumper_ok;
dumper_error:
LOGGER_ERROR("failed to write data, aborting");
dumper_ok:
/* we are done with mysql, close it */
mysql_stmt_close(stmt);
mysql_close(&mysql);
LOGGER_INFO("%s: %zu records, %zu skipped", tablename, count, num_rows_errors);
/* check for mysql errors */
if (mysql_err != MYSQL_NO_DATA) {
LOGGER_ERROR("mysql_stmt_fetch returned an error (code=%d)\n", mysql_err);
err = -1;
}
dumper_close_writer:
/*
* finalize & free memory
*/
for (i = 0; i < n; ++i) {
if (is_var_length_field(field_types[i])) {
size_t ofs = file_writer_wlocpos(&vfields[1][i]);
if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs))))
LOGGER_ERROR("failed to write offset");
file_writer_close(&vfields[0][i]);
file_writer_close(&vfields[1][i]);
} else {
file_writer_close(&ffields[i]);
}
free(bind[i].buffer);
}
vmbuf_free(&buf);
return err;
}
int main(int argc, char *argv[]) {
size_t num_files = 0;
char **files = (char **) calloc(MAX_FILE_SUPPORT, MAX_FILE_SUPPORT * sizeof(char *));
if (0 > init_log_config(&logconf, argc, argv)) {
exit(EXIT_FAILURE);
}
if (!SSTRISEMPTY(logconf.target) && !SSTRISEMPTY(logconf.interface)) {
LOGGER_ERROR("%s", "cannot write to target and interface together. please choose one");
exit(EXIT_FAILURE);
}
char *f = logconf.watch_files;
if (!SSTRISEMPTY(f)) {
while (f != NULL) {
char *fprime = strsep(&f, ",");
if (fprime != NULL) {
files[num_files] = strdup(fprime);
++num_files;
}
}
} else {
LOGGER_ERROR("%s", "no files..no watch!");
exit(EXIT_FAILURE);
}
if (0 > epoll_worker_init()) {
LOGGER_ERROR("%s", "epoll_worker_init");
exit(EXIT_FAILURE);
}
ribs_timer(60*1000, dump_stats);
tab_event_fds = thashtable_create();
delta_push = thashtable_create();
vmbuf_init(&write_buffer, 4096);
vmbuf_init(&mb, 4096);
if (SSTRISEMPTY(logconf.interface) && !SSTRISEMPTY(logconf.target)) {
file_writer_make(&fw);
if (0 > file_writer_init(&fw, logconf.target)) {
LOGGER_ERROR("%s", "flie_writer");
exit(EXIT_FAILURE);
}
write_to_file = true;
} else if (!SSTRISEMPTY(logconf.interface)) {
if (0 > http_client_pool_init(&client_pool, 20, 20)) {
LOGGER_ERROR("http_client_pool_init");
exit(EXIT_FAILURE);
}
memset(&eserv, 0, sizeof(eserv));
vmbuf_reset(&write_buffer);
_replace(logconf.interface, &write_buffer, "http://www.", "");
_replace(logconf.interface, &write_buffer, "http://", "");
char *interface = vmbuf_data(&write_buffer);
eserv.context = ribs_strdup(strstr(interface, "/"));
char *ln = strchr(interface, '/');
ln = ribs_malloc_sprintf("%.*s", ((int)strlen(interface) - (int)strlen(ln)), interface);
if (0 > parse_host_to_inet(ln, eserv.hostname, &eserv.server, &eserv.port)) {
LOGGER_ERROR("%s", "server details invalid. cannot parse server");
exit(EXIT_FAILURE);
}
} else {
LOGGER_ERROR("%s", "no target defined. please use target or interface");
exit(EXIT_FAILURE);
}
char _hostname[1024];
gethostname(_hostname, 1024);
hostname = ribs_strdup(_hostname);
int wd = inotify_init1(IN_NONBLOCK);
if (0 >= wd) {
LOGGER_ERROR("%s", "failed to init inotify. cannot proceed. make sure you've inotify and is accessible to this user.");
exit(EXIT_FAILURE);
}
if (!recursive_flush_events(wd, files, num_files)) {
LOGGER_ERROR("%s", "collection failed");
abort();
}
return 0;
}
static bool
recursive_flush_events (
int inotify_wd,
char **files,
uint32_t num_files) {
struct logz_file_def *filedef = ribs_malloc(num_files * sizeof(struct logz_file_def));
int prev_wd;
size_t evlen = 0;
// size_t evbuf_off = 0;
bool found_unwatchable_dir = false;
bool no_inotify_resources = false;
int inserted = 0;
size_t i;
for (i = 0; i < num_files; i++) {
filedef[i].name = files[i];
size_t fnlen = strlen (filedef[i].name);
if (evlen < fnlen)
evlen = fnlen;
filedef[i].wd = -1;
char *file_fullname = ribs_strdup(filedef[i].name);
char *dir_name = dirname(file_fullname);
size_t dirlen = strlen(dir_name);;
char prev = filedef[i].name[dirlen];
filedef[i].basename_start = basename (file_fullname) - filedef[i].name;
filedef[i].name[dirlen] = '\0';
filedef[i].parent_wd = inotify_add_watch(inotify_wd, dir_name, (IN_CREATE | IN_MOVED_TO));
filedef[i].name[dirlen] = prev;
if(filedef[i].parent_wd < 0) {
if (errno != ENOSPC)
LOGGER_ERROR("cannot watch parent directory of file %s", filedef[i].name);
else {
no_inotify_resources = true;
LOGGER_ERROR("%s", "inotify resources exhausted");
}
found_unwatchable_dir = true;
break;
}
filedef[i].wd = inotify_add_watch(inotify_wd, filedef[i].name, inotify_file_watch_mask);
if (filedef[i].wd < 0) {
if (errno == ENOSPC) {
no_inotify_resources = true;
LOGGER_ERROR("%s", "inotify resources exhausted");
} else if(errno != filedef[i].errnum)
LOGGER_ERROR("cannot watch %s", filedef[i].name);
continue;
}
filedef[i].fd = open(filedef[i].name, O_RDONLY | O_NONBLOCK);
if (0 >= filedef[i].fd) {
LOGGER_ERROR("skipping file %s. cannot open to read", filedef[i].name);
continue;
}
struct stat stats;
if (fstat (filedef[i].fd, &stats) != 0) {
LOGGER_ERROR("skipping file %s.cannot stat", filedef[i].name);
filedef->errnum = errno;
logz_close_fd (filedef[i].fd, filedef[i].name);
filedef->fd = -1;
continue;
}
filedef[i].size = stats.st_size;
lseek (filedef[i].fd, 0, SEEK_END); // no offset enforced
thashtable_insert(tab_event_fds, &filedef[i].wd, sizeof(filedef[i].wd), &filedef[i], sizeof(filedef[i]), &inserted);
struct vmbuf kdelta = VMBUF_INITIALIZER;
vmbuf_init(&kdelta, 4096);
thashtable_insert(delta_push, &filedef[i].fd, sizeof(filedef[i].fd), &kdelta, sizeof(kdelta), &inserted);
}
if(no_inotify_resources || found_unwatchable_dir) {
LOGGER_ERROR("%s", "running low on inotify resources / got an unwatchable directory. Aborting!!");
abort();
}
prev_wd = filedef[num_files -1].wd;
evlen += sizeof (struct inotify_event) + 1;
struct vmbuf evbuf = VMBUF_INITIALIZER;
vmbuf_init(&evbuf, evlen);
ssize_t res = 0;
struct timeval delay; /* how long to wait for file changes. */
delay.tv_sec = (time_t) 0.50;
delay.tv_usec = 1000000 * (0.50 - delay.tv_sec);
fd_set rfd;
FD_ZERO (&rfd);
FD_SET (inotify_wd, &rfd);
while(1) {
if (thashtable_get_size(tab_event_fds) == 0) {
LOGGER_INFO("%s", "no file to read");
return true;
}
{
int file_change = select(inotify_wd + 1, &rfd, NULL, NULL, NULL);
if (file_change == 0)
continue;
else if (file_change == -1) {
LOGGER_ERROR("%s", "error monitoring inotify event");
exit(EXIT_FAILURE);
}
vmbuf_reset(&evbuf);
while (0 < (res = read(inotify_wd, vmbuf_wloc(&evbuf), evlen))) {
if (0 > vmbuf_wseek(&evbuf, res))
return false;
else if (errno == EINTR)
continue;
else if (0 < res)
break;
}
if (errno == EAGAIN && res < 0)
continue;
// res might be 0, or could have overrun memory
if (res == 0) {
LOGGER_ERROR("%s", "error reading inotify event|bad buffer size. aborting to investigate");
abort();
}
// another case when res == 0 or could have overrun memory or got EINVAL
// -- ignored --
// what to do if? .. realloc buffer. but the deal is we're on vmbuf which will grow if world is that bad. we're mostly safe here. hence ignored.
}
struct logz_file_def *tmp;
struct inotify_event *event = (struct inotify_event *)vmbuf_data(&evbuf);
if (event->len) {
// events of lower interest?. these are from watched directory. we'll drop those which we're not watching for and will set watch on those of interest.
size_t x;
for (x = 0; x < num_files; x++) {
if (filedef[x].parent_wd == event->wd
&& strcmp (event->name, filedef[x].name + filedef[x].basename_start))
break;
}
if (x == num_files)
continue;
int wdx = inotify_add_watch (inotify_wd, filedef[x].name, inotify_file_watch_mask);
if (0 > wdx) {
LOGGER_ERROR("cannot watch %s", filedef[x].name);
continue;
}
tmp = &(filedef[x]);
thashtable_remove(tab_event_fds, &tmp->wd, sizeof (tmp->wd));
tmp->wd = wdx;
thashtable_insert(tab_event_fds, &tmp->wd, sizeof(tmp->wd), &tmp, sizeof(struct logz_file_def), &inserted);
// rebalance new found file | make all assertions | we'll read from this as well.
UNUSED(tmp);
} else {
thashtable_rec_t *rec = thashtable_lookup(tab_event_fds, &event->wd, sizeof(event->wd));
tmp = (struct logz_file_def *)thashtable_get_val(rec);
}
if (!tmp) {
continue;
}
if (event->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF)) {
if (event->mask & IN_DELETE_SELF) {
inotify_rm_watch(inotify_wd, tmp->wd);
thashtable_remove(tab_event_fds, &tmp->wd, sizeof(tmp->wd));
}
continue;
}
_flush(tmp, event->wd, &prev_wd);
}
return true;
}
