Exemple #1
0
inline void http_server_write(void) {
    struct http_server_context *ctx = http_server_get_context();
    struct iovec iovec[2] = {
        { vmbuf_data(&ctx->header), vmbuf_wlocpos(&ctx->header)},
        { vmbuf_data(&ctx->payload), vmbuf_wlocpos(&ctx->payload)}
    };
    int fd = ctx->fd;

    ssize_t num_write;
    for (;;http_server_yield()) {
        num_write = writev(fd, iovec, iovec[1].iov_len ? 2 : 1);
        if (0 > num_write) {
            if (EAGAIN == errno) {
                continue;
            } else {
                ctx->persistent = 0;
                return;
            }
        } else {
            if (num_write >= (ssize_t)iovec[0].iov_len) {
                num_write -= iovec[0].iov_len;
                iovec[0].iov_len = iovec[1].iov_len - num_write;
                if (iovec[0].iov_len == 0)
                    break;
                iovec[0].iov_base = iovec[1].iov_base + num_write;
                iovec[1].iov_len = 0;
            } else {
                iovec[0].iov_len -= num_write;
                iovec[0].iov_base += num_write;
            }
        }
    }
}
Exemple #2
0
static void
write_out_stream (const char *filename, char *data) {
    vmbuf_reset(&write_buffer);
    vmbuf_sprintf(&write_buffer, "{ \"message\": \"%s|%s|", hostname, filename);
    json_escape_str_vmb(&write_buffer, data);
    vmbuf_strcpy(&write_buffer, "\" }");
    vmbuf_chrcpy(&write_buffer, '\0');

    if (write_to_file) {
        if (0 > file_writer_write(&fw, vmbuf_data(&write_buffer), vmbuf_wlocpos(&write_buffer))) {
            LOGGER_ERROR("%s", "failed write attempt on outfile| aborting to diagnose!");
            abort();
        }
        return;
    }

    int threshold = INTERFACE_ONERROR_RETRY_THRESHOLD;
    while (0 > post_to_interface(vmbuf_data(&write_buffer), vmbuf_wlocpos(&write_buffer)) && (0 < threshold--)) {
        if (0 == post_to_interface(vmbuf_data(&write_buffer), vmbuf_wlocpos(&write_buffer))) {
            LOGGER_ERROR("post failed to %s, issuing reattempt#%d", eserv.hostname, threshold);
            --failure;
            break;
        }
    }
}
Exemple #3
0
static void
trigger_writer (
    const char *filename,
    struct logz_file_def *filedef) {

    vmbuf_reset(&write_buffer);
    ssize_t res;
    char *fn = basename(ribs_strdup(filename));
    while(1) {
        vmbuf_reset(&write_buffer);
        res = read(filedef->fd, vmbuf_wloc(&write_buffer), (BUFSIZ + 1024) &~ 1024);

        filedef->size += res;
        lseek (filedef->fd, filedef->size, SEEK_SET);

        if (0 > vmbuf_wseek(&write_buffer, res)) {
            LOGGER_ERROR("%s", "wseek error");
            break;
        } else if (0 > res) {
            LOGGER_ERROR("%s", "read error"); // EAGAIN is handled by poller
            break;
        } else if (0 < res) {
            // initial sanitizer
            vmbuf_chrcpy(&write_buffer, '\0'); // kill garbage
            char *data = ribs_strdup(vmbuf_data(&write_buffer));
            //data = strchr(data, '\n') + 1; // skip broken data from initial buffer start. we read from where the file was first observed
            ssize_t write_depth = res = strlen(data);
            // line doesn't end here
            if (data[res - 1] != '\n') {
                char *datafringe = ribs_strdup((char *)memrchr(data, '\n', res));
                if (SSTRISEMPTY(datafringe))
                    break;
                write_depth = strlen(data) - strlen(datafringe);
                *(data + write_depth) = 0;

                if (filedef->size != 0) {
                    char *rebalanced_data = write_file_fringe(fn, data, filedef->fd);
                    if (NULL != rebalanced_data) {
                        data = rebalanced_data;
                        write_depth = strlen(data);
                    }
                }

                thashtable_rec_t *rec = thashtable_lookup(delta_push, &filedef->fd, sizeof(filedef->fd));
                struct vmbuf kdelta = *(struct vmbuf *)thashtable_get_val(rec);
                vmbuf_strcpy(&kdelta, datafringe);
                vmbuf_chrcpy(&kdelta, '\0');
            }

            vmbuf_reset(&write_buffer);
            vmbuf_memcpy(&write_buffer, data, write_depth);
            vmbuf_chrcpy(&write_buffer, '\0');
            write_out_stream(fn, ribs_strdup(vmbuf_data(&write_buffer)));
        } else if (0 == res) {
            break;
        }
    }
}
Exemple #4
0
int mysql_pool_get(struct mysql_login_info *info, struct mysql_pool_entry **mysql) {
    if (0 > make_ht_key(&misc, info))
        return -1;
    char *ht_key = vmbuf_data(&misc);
    size_t key_len = vmbuf_wlocpos(&misc);

    uint32_t ofs = hashtable_lookup(&ht_idle_connections, ht_key, key_len);
    if (0 == ofs) {
        struct list *l = (struct list *)calloc(1, sizeof(struct list));
        list_init(l);
        ofs = hashtable_insert(&ht_idle_connections,
                               ht_key, key_len,
                               &l, sizeof(struct list *));
        if (0 == ofs) // unable to insert
            return -1;
        // add one element since we know there isn't any
        if (0 > create_entry(l))
            return -1;
        // get first free element
        if (0 > get_free_entry(info, l, mysql))
            return -1;
    }
    struct list *l = *(struct list **)hashtable_get_val(&ht_idle_connections, ofs);
    if (list_empty(l)) {
        LOGGER_INFO("adding one more entry in the list");
        if (0 > create_entry(l))
            return -1;
    }
    if (0 > get_free_entry(info, l, mysql))
        return -1;
    return 0;
}
Exemple #5
0
int http_client_pool_post_request_send(struct http_client_context *context, struct vmbuf *post_data) {
    size_t size = vmbuf_wlocpos(post_data);
    vmbuf_sprintf(&context->request, "\r\nContent-Length: %zu\r\n\r\n", size);
    // TODO: use writev instead of copying
    vmbuf_memcpy(&context->request, vmbuf_data(post_data), size);
    if (0 > http_client_send_request(context)) {
        http_client_free(context);
        return -1;
    }
    return 0;
}
Exemple #6
0
int mysql_pool_free(struct mysql_login_info *info, struct mysql_pool_entry *mysql) {
    if (0 > make_ht_key(&misc, info))
        return -1;
    char *ht_key = vmbuf_data(&misc);
    size_t key_len = vmbuf_wlocpos(&misc);

    uint32_t ofs = hashtable_lookup(&ht_idle_connections, ht_key, key_len);
    if (0 == ofs)
        return -1;
    struct list *l = *(struct list **)hashtable_get_val(&ht_idle_connections, ofs);
    list_insert_head(l, &(mysql->l));
    return 0;
}
Exemple #7
0
static char *
write_file_fringe (const char *filename, char *data, int fd) {
    thashtable_rec_t *rec = thashtable_lookup(delta_push, &fd, sizeof(fd));
    struct vmbuf delta = *(struct vmbuf *)thashtable_get_val(rec);
    char *past = vmbuf_data(&delta);
    char *lookahead = NULL;

    if (!SSTRISEMPTY(past)) {
        lookahead = strchr(data, '\n');
        char *trailing_past = ribs_malloc_sprintf("%.*s", ((int)strlen(data) - (int)strlen(lookahead)), data);
        if (trailing_past) {
            char *d_composite = ribs_malloc_sprintf("%s%s", past, trailing_past);
            write_out_stream(filename, d_composite);
            d_composite += strlen(d_composite); // advance by that
            vmbuf_reset(&delta);
        }
    }
    return lookahead;
}
Exemple #8
0
int ds_var_field_writer_close(struct ds_var_field_writer *dsvfw) {
    if (0 > ds_var_field_writer_new_row(dsvfw) ||
        0 > file_writer_align(&dsvfw->data))
        return -1;
    size_t ofs_table = file_writer_wlocpos(&dsvfw->data);
    if (0 > file_writer_write(&dsvfw->data, vmbuf_data(&dsvfw->ofs_table), vmbuf_wlocpos(&dsvfw->ofs_table)))
        return -1;

    size_t ofs_last = file_writer_wlocpos(&dsvfw->data);
    if (0 > file_writer_lseek(&dsvfw->data, 0, SEEK_SET))
        return -1;

    struct ds_var_field_header *header = (struct ds_var_field_header *)file_writer_wloc(&dsvfw->data);
    header->type = -header->type;
    header->ofs_table = ofs_table;
    if (0 > file_writer_lseek(&dsvfw->data, ofs_last, SEEK_SET) ||
        0 > file_writer_close(&dsvfw->data))
        return -1;
    vmbuf_free(&dsvfw->ofs_table);
    return 0;
}
Exemple #9
0
void http_client_fiber_main(void) {
    struct http_client_context *ctx = (struct http_client_context *)current_ctx->reserved;
    int fd = current_ctx->fd;
    struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + fd;
    TIMEOUT_HANDLER_REMOVE_FD_DATA(fd_data);

    int persistent = 0;
    epoll_worker_set_last_fd(fd); /* needed in the case where epoll_wait never occured */

    /*
     * write request
     */
    int res;
    for (; (res = vmbuf_write(&ctx->request, fd)) == 0; http_client_yield());
    if (0 > res) {
        LOGGER_PERROR("write");
        CLIENT_ERROR();
    }
    /*
     * HTTP header
     */
    uint32_t eoh_ofs;
    char *data;
    char *eoh;
    res = vmbuf_read(&ctx->response, fd);
    *vmbuf_wloc(&ctx->response) = 0;
    READ_MORE_DATA_STR(NULL == (eoh = strstr(data = vmbuf_data(&ctx->response), CRLFCRLF)));
    eoh_ofs = eoh - data + SSTRLEN(CRLFCRLF);
    *eoh = 0;
    char *p = strstr(data, CONNECTION);
    if (p != NULL) {
        p += SSTRLEN(CONNECTION);
        persistent = (0 == SSTRNCMPI(CONNECTION_CLOSE, p) ? 0 : 1);
    }
    SSTRL(HTTP, "HTTP/");
    if (0 != SSTRNCMP(HTTP, data))
        CLIENT_ERROR();

    p = strchrnul(data, ' ');
    int code = (*p ? atoi(p + 1) : 0);
    if (0 == code)
        CLIENT_ERROR();
    do {
        if (code == 204 || code == 304) /* No Content,  Not Modified */
            break;
        /*
         * content length
         */
        char *content_len_str = strstr(data, CONTENT_LENGTH);
        if (NULL != content_len_str) {
            content_len_str += SSTRLEN(CONTENT_LENGTH);
            size_t content_end = eoh_ofs + atoi(content_len_str);
            READ_MORE_DATA(vmbuf_wlocpos(&ctx->response) < content_end);
            break;
        }
        /*
         * chunked encoding
         */
        char *transfer_encoding_str = strstr(data, TRANSFER_ENCODING);
        if (NULL != transfer_encoding_str &&
                0 == SSTRNCMP(transfer_encoding_str + SSTRLEN(TRANSFER_ENCODING), "chunked")) {
            size_t chunk_start = eoh_ofs;
            size_t data_start = eoh_ofs;
            char *p;
            for (;;) {
                READ_MORE_DATA_STR(*(p = strchrnul((data = vmbuf_data(&ctx->response)) + chunk_start, '\r')) == 0);
                if (0 != SSTRNCMP(CRLF, p))
                    CLIENT_ERROR();
                uint32_t s = strtoul(data + chunk_start, NULL, 16);
                if (0 == s) {
                    vmbuf_wlocset(&ctx->response, data_start);
                    break;
                }
                chunk_start = p - data + SSTRLEN(CRLF);
                size_t chunk_end = chunk_start + s + SSTRLEN(CRLF);
                READ_MORE_DATA(vmbuf_wlocpos(&ctx->response) < chunk_end);
                memmove(vmbuf_data(&ctx->response) + data_start, vmbuf_data(&ctx->response) + chunk_start, s);
                data_start += s;
                chunk_start = chunk_end;
            }
            break;
        }
        /*
         * older versions of HTTP, terminated by disconnect
         */
        for (;; yield()) {
            if ((res = vmbuf_read(&ctx->response, fd)) < 0)
                CLIENT_ERROR();
            if (0 == res)
                break; /* remote side closed connection */
        }
    } while (0);
    ctx->content = vmbuf_data_ofs(&ctx->response, eoh_ofs);
    ctx->content_length = vmbuf_wlocpos(&ctx->response) - eoh_ofs;
    ctx->http_status_code = code;
    vmbuf_data_ofs(&ctx->response, eoh_ofs - SSTRLEN(CRLFCRLF))[0] = CR;
    *vmbuf_wloc(&ctx->response) = 0;
    ctx->persistent = persistent;
    if (!persistent)
        close(fd);
}
Exemple #10
0
void http_server_fiber_main(void) {
    struct http_server_context *ctx = http_server_get_context();
    struct http_server *server = ctx->server;
    int fd = ctx->fd;

    char *URI;
    char *headers;
    char *content;
    size_t content_length;
    int res;
    ctx->persistent = 0;

    vmbuf_init(&ctx->request, server->init_request_size);
    vmbuf_init(&ctx->header, server->init_header_size);
    vmbuf_init(&ctx->payload, server->init_payload_size);
    size_t max_req_size = server->max_req_size;

    for (;; http_server_yield()) {
        READ_FROM_SOCKET();
        if (vmbuf_wlocpos(&ctx->request) > MIN_HTTP_REQ_SIZE)
            break;
    }
    do {
        if (0 == SSTRNCMP(GET, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(HEAD, vmbuf_data(&ctx->request))) {
            /* GET or HEAD */
            while (0 != SSTRNCMP(CRLFCRLF,  vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF))) {
                http_server_yield();
                READ_FROM_SOCKET();
            }
            /* make sure the string is \0 terminated */
            /* this will overwrite the first CR */
            *(vmbuf_wloc(&ctx->request) - SSTRLEN(CRLFCRLF)) = 0;
            char *p = vmbuf_data(&ctx->request);
            ctx->persistent = check_persistent(p);
            URI = strchrnul(p, ' '); /* can't be NULL GET and HEAD constants have space at the end */
            *URI = 0;
            ++URI; // skip the space
            p = strchrnul(URI, '\r'); /* HTTP/1.0 */
            headers = p;
            if (0 != *headers) /* are headers present? */
                headers += SSTRLEN(CRLF); /* skip the new line */
            *p = 0;
            p = strchrnul(URI, ' '); /* truncate the version part */
            *p = 0; /* \0 at the end of URI */

            ctx->content = NULL;
            ctx->content_len = 0;

            /* minimal parsing and call user function */
            http_server_process_request(URI, headers);
        } else if (0 == SSTRNCMP(POST, vmbuf_data(&ctx->request)) || 0 == SSTRNCMP(PUT, vmbuf_data(&ctx->request))) {
            /* POST or PUT */
            for (;;) {
                *vmbuf_wloc(&ctx->request) = 0;
                /* wait until we have the header */
                if (NULL != (content = strstr(vmbuf_data(&ctx->request), CRLFCRLF)))
                    break;
                http_server_yield();
                READ_FROM_SOCKET();
            }
            *content = 0; /* terminate at the first CR like in GET */
            content += SSTRLEN(CRLFCRLF);
            size_t content_ofs = content - vmbuf_data(&ctx->request);

            if (strstr(vmbuf_data(&ctx->request), EXPECT_100)) {
                vmbuf_sprintf(&ctx->header, "%s %s\r\n\r\n", HTTP_SERVER_VER, HTTP_STATUS_100);
                if (0 > vmbuf_write(&ctx->header, fd)) {
                    close(fd);
                    return;
                }
                vmbuf_reset(&ctx->header);
            }
            ctx->persistent = check_persistent(vmbuf_data(&ctx->request));

            /* parse the content length */
            char *p = strcasestr(vmbuf_data(&ctx->request), CONTENT_LENGTH);
            if (NULL == p) {
                http_server_response(HTTP_STATUS_411, HTTP_CONTENT_TYPE_TEXT_PLAIN);
                break;
            }

            p += SSTRLEN(CONTENT_LENGTH);
            content_length = atoi(p);
            for (;;) {
                if (content_ofs + content_length <= vmbuf_wlocpos(&ctx->request))
                    break;
                http_server_yield();
                READ_FROM_SOCKET();
            }
            p = vmbuf_data(&ctx->request);
            URI = strchrnul(p, ' '); /* can't be NULL PUT and POST constants have space at the end */
            *URI = 0;
            ++URI; /* skip the space */
            p = strchrnul(URI, '\r'); /* HTTP/1.0 */
            headers = p;
            if (0 != *headers) /* are headers present? */
                headers += SSTRLEN(CRLF); /* skip the new line */
            *p = 0;
            p = strchrnul(URI, ' '); /* truncate http version */
            *p = 0; /* \0 at the end of URI */
            ctx->content = vmbuf_data_ofs(&ctx->request, content_ofs);
            *(ctx->content + content_length) = 0;
            ctx->content_len = content_length;

            /* minimal parsing and call user function */
            http_server_process_request(URI, headers);
        } else {
            http_server_response(HTTP_STATUS_501, HTTP_CONTENT_TYPE_TEXT_PLAIN);
            break;
        }
    } while(0);

    if (vmbuf_wlocpos(&ctx->header) > 0) {
        epoll_worker_resume_events(fd);
        http_server_write();
    }

    if (ctx->persistent) {
        struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + fd;
        fd_data->ctx = server->idle_ctx;
        timeout_handler_add_fd_data(&server->timeout_handler, fd_data);
    } else
        close(fd);
}
Exemple #11
0
int mysql_dumper_dump(struct mysql_login_info *mysql_login_info, const char *outputdir, const char *dbname, const char *tablename, const char *query, size_t query_len, struct mysql_dumper_type *types) {
    MYSQL mysql;
    MYSQL_STMT *stmt = NULL;
    mysql_init(&mysql);
    my_bool b_flag = 1;
    if (0 != mysql_options(&mysql, MYSQL_OPT_RECONNECT, (const char *)&b_flag))
        return report_error(&mysql);

    if (NULL == mysql_real_connect(&mysql, mysql_login_info->host, mysql_login_info->user, mysql_login_info->pass, mysql_login_info->db, mysql_login_info->port, NULL, CLIENT_COMPRESS))
        return report_error(&mysql);

    b_flag = 0;
    if (0 != mysql_options(&mysql, MYSQL_REPORT_DATA_TRUNCATION, (const char *)&b_flag))
        return report_error(&mysql);

    stmt = mysql_stmt_init(&mysql);
    if (!stmt)
        return report_error(&mysql);

    if (0 != mysql_stmt_prepare(stmt, query, query_len))
        return report_stmt_error(&mysql, stmt);

    MYSQL_RES *rs = mysql_stmt_result_metadata(stmt);
    if (!rs)
        return report_stmt_error(&mysql, stmt);

    unsigned int n = mysql_num_fields(rs);
    MYSQL_FIELD *fields = mysql_fetch_fields(rs);
    int field_types[n];
    MYSQL_BIND bind[n];
    my_bool is_null[n];
    unsigned long length[n];
    my_bool error[n];
    memset(bind, 0, sizeof(MYSQL_BIND) * n);
    int null_terminate_str[n];
    memset(null_terminate_str, 0, sizeof(int) * n);

    struct file_writer ffields[n];
    struct file_writer vfields[2][n];

    struct vmbuf buf = VMBUF_INITIALIZER;
    vmbuf_init(&buf, 4096);
    vmbuf_sprintf(&buf, "%s/%s/%s/schema.txt", outputdir, dbname, tablename);
    mkdir_for_file_recursive(vmbuf_data(&buf));
    int fdschema = creat(vmbuf_data(&buf), 0644);
    struct vmfile ds_txt = VMFILE_INITIALIZER;
    vmbuf_reset(&buf);
    vmbuf_sprintf(&buf, "%s/%s/%s/ds.txt", outputdir, dbname, tablename);
    if (0 > vmfile_init(&ds_txt, vmbuf_data(&buf), 4096))
        return LOGGER_ERROR("failed to create: %s", vmbuf_data(&buf)), vmbuf_free(&buf), -1;
    vmfile_sprintf(&ds_txt, "DS_LOADER_BEGIN()\n");
    vmfile_sprintf(&ds_txt, "/*\n * DB: %s\n */\n", dbname);
    vmfile_sprintf(&ds_txt, "#undef DB_NAME\n#define DB_NAME %s\n", dbname);
    ssize_t len = 80 - strlen(tablename);
    if (len < 0) len = 4;
    char header[len];
    memset(header, '=', len);
    vmfile_sprintf(&ds_txt, "/* %.*s[ %s ]%.*s */\n", (int)len / 2, header, tablename, (int)(len - (len / 2)), header);
    vmfile_sprintf(&ds_txt, "#   undef TABLE_NAME\n#   define TABLE_NAME %s\n", tablename);
    /*
     * initialize output files
     */
    unsigned int i;
    for (i = 0; i < n; ++i) {
        file_writer_make(&ffields[i]);
        file_writer_make(&vfields[0][i]);
        file_writer_make(&vfields[1][i]);
    }

    struct hashtable ht_types = HASHTABLE_INITIALIZER;
    hashtable_init(&ht_types, 32);
    if (NULL != types) {
        struct mysql_dumper_type *t = types;
        for (; t->name; ++t) {
            /* storing ptr here which points to static str */
            hashtable_insert(&ht_types, t->name, strlen(t->name), t, sizeof(struct mysql_dumper_type));
        }
    }
    /*
     * parse meta data and construct bind array
     */
    int err = 0;
    for (i = 0; i < n; ++i) {
        field_types[i] = fields[i].type;
        bind[i].is_unsigned = IS_UNSIGNED(fields[i].flags);

        int64_t ds_type = -1;
        const char *ds_type_str = "VAR";
        /*
         * handle overrides
         */
        while (NULL != types) {
            uint32_t ofs = hashtable_lookup(&ht_types, fields[i].name, strlen(fields[i].name));
            if (!ofs) break;
            struct mysql_dumper_type *type = (struct mysql_dumper_type *)hashtable_get_val(&ht_types, ofs);
            null_terminate_str[i] = MYSQL_DUMPER_CSTR & type->flags;
            if (type->mysql_type)
                field_types[i] = type->mysql_type;
            bind[i].is_unsigned = (type->flags & MYSQL_DUMPER_UNSIGNED) > 0 ? 1 : 0;
            break;
        }
        vmbuf_reset(&buf);
        vmbuf_sprintf(&buf, "%s/%s/%s/%s", outputdir, dbname, tablename, fields[i].name);

        mkdir_for_file_recursive(vmbuf_data(&buf));
        if (is_var_length_field(field_types[i])) {
            size_t ofs = vmbuf_wlocpos(&buf);
            vmbuf_sprintf(&buf, ".ofs");
            if (0 > (err = file_writer_init(&vfields[0][i], vmbuf_data(&buf))))
                break;
            vmbuf_wlocset(&buf, ofs);
            vmbuf_sprintf(&buf, ".dat");
            if (0 > (err = file_writer_init(&vfields[1][i], vmbuf_data(&buf))))
                break;
        } else {
            ds_type = get_ds_type(field_types[i], bind[i].is_unsigned);
            const char *s = get_ds_type_str(ds_type);
            if (*s) ds_type_str = s;
            if (0 > (err = file_writer_init(&ffields[i], vmbuf_data(&buf))) ||
                0 > (err = file_writer_write(&ffields[i], &ds_type, sizeof(ds_type))))
                break;;
        }
        len = ribs_mysql_get_storage_size(field_types[i], fields[i].length);
        if (fdschema > 0)
            dprintf(fdschema, "%03d  name = %s, size=%zu, length=%lu, type=%s (%s), is_prikey=%d, ds_type=%s\n", i, fields[i].name, len, fields[i].length, ribs_mysql_get_type_name(field_types[i]), bind[i].is_unsigned ? "unsigned" : "signed", IS_PRI_KEY(fields[i].flags), ds_type_str);
        if (is_var_length_field(field_types[i])) {
            vmfile_sprintf(&ds_txt, "        DS_VAR_FIELD_LOADER(%s)\n", fields[i].name);
        } else {
            vmfile_sprintf(&ds_txt, "        DS_FIELD_LOADER(%s, %s)\n", ds_type_str, fields[i].name);
        }
        bind[i].buffer_type = field_types[i];
        bind[i].buffer_length = len;
        bind[i].buffer = malloc(len);
        bind[i].is_null = &is_null[i];
        bind[i].length = &length[i];
        bind[i].error = &error[i];
    }
    hashtable_free(&ht_types);
    mysql_free_result(rs);
    close(fdschema);
    //vmfile_sprintf(&ds_txt, "/*\n * TABLE END: %s\n */\n", tablename);
    vmfile_sprintf(&ds_txt, "DS_LOADER_END()\n");
    vmfile_close(&ds_txt);
    /*
     * execute & bind
     */
    if (0 != err ||
        0 != mysql_stmt_execute(stmt) ||
        0 != mysql_stmt_bind_result(stmt, bind)) {
        err = -1;
        report_stmt_error(&mysql, stmt);
        goto dumper_close_writer;
    }
    char zeros[4096];
    memset(zeros, 0, sizeof(zeros));
    int mysql_err = 0;
    size_t count = 0, num_rows_errors = 0;
    /*
     * write all rows to output files
     */
    while (0 == (mysql_err = mysql_stmt_fetch(stmt))) {
        int b = 0;
        for (i = 0; i < n && !b; ++i)
            b = b || error[i];
        if (b) {
            ++num_rows_errors;
            continue;
        }
        for (i = 0; i < n; ++i) {
            if (is_var_length_field(field_types[i])) {
                size_t ofs = file_writer_wlocpos(&vfields[1][i]);
                if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs))) ||
                    0 > (err = file_writer_write(&vfields[1][i], is_null[i] ? NULL : bind[i].buffer, is_null[i] ? 0 : length[i])))
                    goto dumper_error;
                if (null_terminate_str[i]) {
                    const char c = '\0';
                    if (0 > (err = file_writer_write(&vfields[1][i], &c, sizeof(c))))
                        goto dumper_error;
                }
            } else {
                if (0 > (err = file_writer_write(&ffields[i], is_null[i] ? zeros : bind[i].buffer, bind[i].buffer_length)))
                    goto dumper_error;
            }
        }
        ++count;
    }
    /* no dumper errors */
    goto dumper_ok;
 dumper_error:
    LOGGER_ERROR("failed to write data, aborting");
 dumper_ok:
    /* we are done with mysql, close it */
    mysql_stmt_close(stmt);
    mysql_close(&mysql);
    LOGGER_INFO("%s: %zu records, %zu skipped", tablename, count, num_rows_errors);
    /* check for mysql errors */
    if (mysql_err != MYSQL_NO_DATA) {
        LOGGER_ERROR("mysql_stmt_fetch returned an error (code=%d)\n", mysql_err);
        err = -1;
    }
 dumper_close_writer:
    /*
     * finalize & free memory
     */
    for (i = 0; i < n; ++i) {
        if (is_var_length_field(field_types[i])) {
            size_t ofs = file_writer_wlocpos(&vfields[1][i]);
            if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs))))
                LOGGER_ERROR("failed to write offset");
            file_writer_close(&vfields[0][i]);
            file_writer_close(&vfields[1][i]);
        } else {
            file_writer_close(&ffields[i]);
        }
        free(bind[i].buffer);
    }
    vmbuf_free(&buf);
    return err;
}
Exemple #12
0
int main(int argc, char *argv[]) {

    size_t num_files = 0;

    char **files = (char **) calloc(MAX_FILE_SUPPORT, MAX_FILE_SUPPORT * sizeof(char *));

    if (0 > init_log_config(&logconf, argc, argv)) {
        exit(EXIT_FAILURE);
    }
    if (!SSTRISEMPTY(logconf.target) && !SSTRISEMPTY(logconf.interface)) {
        LOGGER_ERROR("%s", "cannot write to target and interface together. please choose one");
        exit(EXIT_FAILURE);
    }
        

    char *f = logconf.watch_files;
    if (!SSTRISEMPTY(f)) {
        while (f != NULL) {
            char *fprime = strsep(&f, ",");
            if (fprime != NULL) {
                files[num_files] = strdup(fprime);
                ++num_files;
            }
        }
    } else {
        LOGGER_ERROR("%s", "no files..no watch!");
        exit(EXIT_FAILURE);
    }

    if (0 > epoll_worker_init()) {
        LOGGER_ERROR("%s", "epoll_worker_init");
        exit(EXIT_FAILURE);
    }

    ribs_timer(60*1000, dump_stats);

    tab_event_fds = thashtable_create();
    delta_push    = thashtable_create();
    vmbuf_init(&write_buffer, 4096);
    vmbuf_init(&mb, 4096);


    if (SSTRISEMPTY(logconf.interface) && !SSTRISEMPTY(logconf.target)) {
        file_writer_make(&fw);

        if (0 > file_writer_init(&fw, logconf.target)) {
            LOGGER_ERROR("%s", "flie_writer");
            exit(EXIT_FAILURE);
        }
        write_to_file = true;
    } else if (!SSTRISEMPTY(logconf.interface)) {

        if (0 > http_client_pool_init(&client_pool, 20, 20)) {
            LOGGER_ERROR("http_client_pool_init");
            exit(EXIT_FAILURE);
        }

        memset(&eserv, 0, sizeof(eserv));

        vmbuf_reset(&write_buffer);
        _replace(logconf.interface, &write_buffer, "http://www.", "");
        _replace(logconf.interface, &write_buffer, "http://", "");

        char *interface = vmbuf_data(&write_buffer);
        eserv.context = ribs_strdup(strstr(interface, "/"));
        
        char *ln = strchr(interface, '/');
        ln = ribs_malloc_sprintf("%.*s", ((int)strlen(interface) - (int)strlen(ln)), interface);

        if (0 > parse_host_to_inet(ln, eserv.hostname, &eserv.server, &eserv.port)) {
            LOGGER_ERROR("%s", "server details invalid. cannot parse server");
            exit(EXIT_FAILURE);
        }
    } else {
        LOGGER_ERROR("%s", "no target defined. please use target or interface");
        exit(EXIT_FAILURE);
    }


    char _hostname[1024];
    gethostname(_hostname, 1024);
    hostname = ribs_strdup(_hostname);

    int wd = inotify_init1(IN_NONBLOCK);
    if (0 >= wd) {
        LOGGER_ERROR("%s", "failed to init inotify. cannot proceed. make sure you've inotify and is accessible to this user.");
        exit(EXIT_FAILURE);
    }

    if (!recursive_flush_events(wd, files, num_files)) {
        LOGGER_ERROR("%s", "collection failed");
        abort();
    }

    return 0;
}
Exemple #13
0
static bool
recursive_flush_events (
    int inotify_wd,
    char **files,
    uint32_t num_files) {

    struct logz_file_def *filedef = ribs_malloc(num_files * sizeof(struct logz_file_def));

    int prev_wd;
    size_t evlen = 0;
    // size_t evbuf_off = 0;
    
    bool found_unwatchable_dir = false;
    bool no_inotify_resources = false;

    int inserted = 0;

    size_t i;
    for (i = 0; i < num_files; i++) {

        filedef[i].name = files[i];
        size_t fnlen = strlen (filedef[i].name);
        if (evlen < fnlen)
            evlen = fnlen;

        filedef[i].wd = -1;
        char *file_fullname = ribs_strdup(filedef[i].name);
        char *dir_name = dirname(file_fullname);
        size_t dirlen = strlen(dir_name);;
        char prev = filedef[i].name[dirlen];
        filedef[i].basename_start = basename (file_fullname) - filedef[i].name;

        filedef[i].name[dirlen] = '\0';

        filedef[i].parent_wd = inotify_add_watch(inotify_wd, dir_name, (IN_CREATE | IN_MOVED_TO));

        filedef[i].name[dirlen] = prev;

        if(filedef[i].parent_wd < 0) {
            if (errno != ENOSPC)
                LOGGER_ERROR("cannot watch parent directory of file %s", filedef[i].name);
            else {
                no_inotify_resources = true;
                LOGGER_ERROR("%s", "inotify resources exhausted");
            }
            found_unwatchable_dir = true;
            break;
        }

        filedef[i].wd = inotify_add_watch(inotify_wd, filedef[i].name, inotify_file_watch_mask);

        if (filedef[i].wd < 0) {
            if (errno == ENOSPC) {
                no_inotify_resources = true;
                LOGGER_ERROR("%s", "inotify resources exhausted");
            } else if(errno != filedef[i].errnum)
                LOGGER_ERROR("cannot watch %s", filedef[i].name);
            continue;
        }
        filedef[i].fd = open(filedef[i].name, O_RDONLY | O_NONBLOCK);
        if (0 >= filedef[i].fd) {
            LOGGER_ERROR("skipping file %s. cannot open to read", filedef[i].name);
            continue;
        }

        struct stat stats;
        if (fstat (filedef[i].fd, &stats) != 0) {
            LOGGER_ERROR("skipping file %s.cannot stat", filedef[i].name);
            filedef->errnum = errno;
            logz_close_fd (filedef[i].fd, filedef[i].name);
            filedef->fd = -1;
            continue;
        }
        filedef[i].size = stats.st_size;
        lseek (filedef[i].fd, 0, SEEK_END); // no offset enforced

        thashtable_insert(tab_event_fds, &filedef[i].wd, sizeof(filedef[i].wd), &filedef[i], sizeof(filedef[i]), &inserted);

        struct vmbuf kdelta = VMBUF_INITIALIZER;
        vmbuf_init(&kdelta, 4096);

        thashtable_insert(delta_push, &filedef[i].fd, sizeof(filedef[i].fd), &kdelta, sizeof(kdelta), &inserted);
    }

    if(no_inotify_resources || found_unwatchable_dir) {
        LOGGER_ERROR("%s", "running low on inotify resources / got an unwatchable directory. Aborting!!");
        abort();
    }

    prev_wd = filedef[num_files -1].wd;

    evlen += sizeof (struct inotify_event) + 1;
    struct vmbuf evbuf = VMBUF_INITIALIZER;
    vmbuf_init(&evbuf, evlen);

    ssize_t res = 0;
    struct timeval delay; /* how long to wait for file changes.  */
    delay.tv_sec = (time_t) 0.50;
    delay.tv_usec = 1000000 * (0.50 - delay.tv_sec);

    fd_set rfd;
    FD_ZERO (&rfd);
    FD_SET (inotify_wd, &rfd);

    while(1) {
        if (thashtable_get_size(tab_event_fds) == 0) {
            LOGGER_INFO("%s", "no file to read");
            return true;
        }

        {
            int file_change = select(inotify_wd + 1, &rfd, NULL, NULL, NULL);

            if (file_change == 0)
                continue;
            else if (file_change == -1) {
                LOGGER_ERROR("%s", "error monitoring inotify event");
                exit(EXIT_FAILURE);
            }

            vmbuf_reset(&evbuf);

            while (0 < (res = read(inotify_wd, vmbuf_wloc(&evbuf), evlen))) {
                if (0 > vmbuf_wseek(&evbuf, res))
                    return false;
                else if (errno == EINTR)
                    continue;
                else if (0 < res)
                    break;
            }
            if (errno == EAGAIN && res < 0)
                continue;
            // res might be 0, or could have overrun memory
            if (res == 0) {
                LOGGER_ERROR("%s", "error reading inotify event|bad buffer size. aborting to investigate");
                abort();
            }
            // another case when res == 0 or could have overrun memory or got EINVAL
            // -- ignored --
            // what to do if? .. realloc buffer. but the deal is we're on vmbuf which will grow if world is that bad. we're mostly safe here. hence ignored.
        }

        struct logz_file_def *tmp;
        struct inotify_event *event = (struct inotify_event *)vmbuf_data(&evbuf);
        if (event->len) {
            // events of lower interest?. these are from watched directory. we'll drop those which we're not watching for and will set watch on those of interest.
            size_t x;
            for (x = 0; x < num_files; x++) {
                if (filedef[x].parent_wd == event->wd
                    && strcmp (event->name, filedef[x].name + filedef[x].basename_start))
                    break;
            }
            if (x == num_files)
                continue;
            int wdx = inotify_add_watch (inotify_wd, filedef[x].name, inotify_file_watch_mask);
            if (0 > wdx) {
                LOGGER_ERROR("cannot watch %s", filedef[x].name);
                continue;
            }
            tmp = &(filedef[x]);
            thashtable_remove(tab_event_fds, &tmp->wd, sizeof (tmp->wd));
            tmp->wd = wdx;
            thashtable_insert(tab_event_fds, &tmp->wd, sizeof(tmp->wd), &tmp, sizeof(struct logz_file_def), &inserted);
            // rebalance new found file | make all assertions | we'll read from this as well.
            UNUSED(tmp);
        } else {
            thashtable_rec_t *rec = thashtable_lookup(tab_event_fds, &event->wd, sizeof(event->wd));
            tmp = (struct logz_file_def *)thashtable_get_val(rec);
        }
        if (!tmp) {
            continue;
        }

        if (event->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF)) {
            if (event->mask & IN_DELETE_SELF) {
                inotify_rm_watch(inotify_wd, tmp->wd);
                thashtable_remove(tab_event_fds, &tmp->wd, sizeof(tmp->wd));
            }
            continue;
        }
        _flush(tmp, event->wd, &prev_wd);
    }
    return true;
}