Esempio n. 1
0
int mysql_pool_get(struct mysql_login_info *info, struct mysql_pool_entry **mysql) {
    if (0 > make_ht_key(&misc, info))
        return -1;
    char *ht_key = vmbuf_data(&misc);
    size_t key_len = vmbuf_wlocpos(&misc);

    uint32_t ofs = hashtable_lookup(&ht_idle_connections, ht_key, key_len);
    if (0 == ofs) {
        struct list *l = (struct list *)calloc(1, sizeof(struct list));
        list_init(l);
        ofs = hashtable_insert(&ht_idle_connections,
                               ht_key, key_len,
                               &l, sizeof(struct list *));
        if (0 == ofs) // unable to insert
            return -1;
        // add one element since we know there isn't any
        if (0 > create_entry(l))
            return -1;
        // get first free element
        if (0 > get_free_entry(info, l, mysql))
            return -1;
    }
    struct list *l = *(struct list **)hashtable_get_val(&ht_idle_connections, ofs);
    if (list_empty(l)) {
        LOGGER_INFO("adding one more entry in the list");
        if (0 > create_entry(l))
            return -1;
    }
    if (0 > get_free_entry(info, l, mysql))
        return -1;
    return 0;
}
Esempio n. 2
0
int mysql_pool_free(struct mysql_login_info *info, struct mysql_pool_entry *mysql) {
    if (0 > make_ht_key(&misc, info))
        return -1;
    char *ht_key = vmbuf_data(&misc);
    size_t key_len = vmbuf_wlocpos(&misc);

    uint32_t ofs = hashtable_lookup(&ht_idle_connections, ht_key, key_len);
    if (0 == ofs)
        return -1;
    struct list *l = *(struct list **)hashtable_get_val(&ht_idle_connections, ofs);
    list_insert_head(l, &(mysql->l));
    return 0;
}
Esempio n. 3
0
const char *mime_types_by_ext(const char *ext) {
    const char *res = DEFAULT_MIME_TYPE;
    size_t n = strlen(ext);
    char tmpext[n];
    char *p = tmpext;
    for (; *ext; ++ext)
        *p++ = tolower(*ext);
    // tmpext doesn't have to be \0 terminated
    size_t ofs = hashtable_lookup(&ht_mime_types, tmpext, n);
    if (ofs)
        return hashtable_get_val(&ht_mime_types, ofs);
    return res;
}
Esempio n. 4
0
static inline void add_cookie(struct hashtable *ht, const char *str) {
    const char *name = str;
    const char *name_end = strchrnul(str, '=');
    const char *val = *name_end ? name_end + 1 : name_end;

    /* get rid of quotes */
    const char *val_st = val, *val_st_end = val + strlen(val);
    if ('"' == *val_st)
        ++val_st;
    if (val_st != val_st_end && '"' == *(val_st_end - 1))
        --val_st_end;

    size_t l = val_st_end - val_st;
    uint32_t loc = hashtable_insert(ht, name, name_end - name, val_st, l + 1);
    *(char *)(hashtable_get_val(ht, loc) + l) = 0;
}
Esempio n. 5
0
struct http_client_context *http_client_pool_create_client(struct http_client_pool *http_client_pool, struct in_addr addr, uint16_t port, struct ribs_context *rctx) {
    int cfd;
    struct http_client_key key = { .addr = addr, .port = port };
    uint32_t ofs = hashtable_lookup(&ht_persistent_clients, &key, sizeof(struct http_client_key));
    struct list *head;
    if (ofs > 0 && !list_empty(head = client_heads + *(uint32_t *)hashtable_get_val(&ht_persistent_clients, ofs))) {
        struct list *client = list_pop_head(head);
        cfd = client - client_chains;
    } else {
        cfd = socket(PF_INET, SOCK_STREAM | SOCK_NONBLOCK, IPPROTO_TCP);
        if (0 > cfd)
            return LOGGER_PERROR("socket"), NULL;

        const int option = 1;
        if (0 > setsockopt(cfd, SOL_SOCKET, SO_REUSEADDR, &option, sizeof(option)))
            return LOGGER_PERROR("setsockopt SO_REUSEADDR"), close(cfd), NULL;

        if (0 > setsockopt(cfd, IPPROTO_TCP, TCP_NODELAY, &option, sizeof(option)))
            return LOGGER_PERROR("setsockopt TCP_NODELAY"), close(cfd), NULL;

        struct sockaddr_in saddr = { .sin_family = AF_INET, .sin_port = htons(port), .sin_addr = addr };
        if (0 > connect(cfd, (struct sockaddr *)&saddr, sizeof(saddr)) && EINPROGRESS != errno)
            return LOGGER_PERROR("connect"), close(cfd), NULL;

        struct epoll_event ev;
        ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
        ev.data.fd = cfd;
        if (0 > epoll_ctl(ribs_epoll_fd, EPOLL_CTL_ADD, cfd, &ev))
            return LOGGER_PERROR("epoll_ctl"), close(cfd), NULL;
    }
    struct ribs_context *new_ctx = ctx_pool_get(&http_client_pool->ctx_pool);
    new_ctx->fd = cfd;
    new_ctx->data.ptr = http_client_pool;
    struct epoll_worker_fd_data *fd_data = epoll_worker_fd_map + cfd;
    fd_data->ctx = new_ctx;
    ribs_makecontext(new_ctx, rctx ? rctx : current_ctx, http_client_fiber_main);
    struct http_client_context *cctx = (struct http_client_context *)new_ctx->reserved;
    cctx->key = (struct http_client_key) {
        .addr = addr, .port = port
    };
    vmbuf_init(&cctx->request, 4096);
    vmbuf_init(&cctx->response, 4096);
    timeout_handler_add_fd_data(&http_client_pool->timeout_handler, fd_data);
    return cctx;
}
Esempio n. 6
0
void http_client_free(struct http_client_pool *client_pool, struct http_client_context *cctx) {
    if (cctx->persistent) {
        int fd = RIBS_RESERVED_TO_CONTEXT(cctx)->fd;
        epoll_worker_set_fd_ctx(fd, idle_ctx);
        uint32_t ofs = hashtable_lookup(&ht_persistent_clients, &cctx->key, sizeof(struct http_client_key));
        struct list *head;
        if (0 == ofs) {
            if (list_empty(&free_list)) {
                close(fd);
                return;
            }
            head = list_pop_head(&free_list);
            uint32_t h = head - client_heads;
            hashtable_insert(&ht_persistent_clients, &cctx->key, sizeof(struct http_client_key), &h, sizeof(h));
            list_init(head);
        } else
            head = client_heads + *(uint32_t *)hashtable_get_val(&ht_persistent_clients, ofs);
        struct list *client = client_chains + fd;
        list_insert_head(head, client);
    }
    ctx_pool_put(&client_pool->ctx_pool, RIBS_RESERVED_TO_CONTEXT(cctx));
}
Esempio n. 7
0
static void _handle_sig_child(void) {
    struct signalfd_siginfo sfd_info;
    for (;;yield()) {
        for (;;) {
            ssize_t res = read(sigfd, &sfd_info, sizeof(struct signalfd_siginfo));
            if (0 > res) {
                if (errno != EAGAIN)
                    LOGGER_PERROR("read from signal fd");
                break;
            }
            if (sizeof(struct signalfd_siginfo) != res) {
                LOGGER_ERROR("failed to read from signal fd, incorrect size");
                continue;
            }
            memset(&last_sig_info, 0, sizeof(last_sig_info));
            epoll_worker_ignore_events(sigfd);
            for (;;) {
                last_sig_info.si_pid = 0;
                if (0 > waitid(P_ALL, 0, &last_sig_info, WEXITED | WNOHANG) && ECHILD != errno) {
                    LOGGER_PERROR("waitid");
                    break;
                }
                pid_t pid = last_sig_info.si_pid;
                if (0 == pid)
                    break; /* no more events */
                if (logger_pid == pid) {
                    char fname[256];
                    snprintf(fname, sizeof(fname), "%s-%d.crash.log", program_invocation_short_name, getpid());
                    fname[sizeof(fname)-1] = 0;
                    int fd = creat(fname, 0644);
                    if (0 <= fd) {
                        dup2(fd, STDOUT_FILENO);
                        dup2(fd, STDERR_FILENO);
                        close(fd);
                        LOGGER_ERROR("logger [%d] terminated unexpectedly: %s, status=%d", pid, _get_exit_reason(&last_sig_info), last_sig_info.si_status);
                    }
                    epoll_worker_exit();
                }
                int i;
                for (i = 0; i < num_instances-1; ++i) {
                    /* check if it is our pid */
                    if (children_pids[i] == pid) {
                        children_pids[i] = 0; /* mark pid as handled */
                        LOGGER_ERROR("child process [%d] terminated unexpectedly: %s, status=%d", pid, _get_exit_reason(&last_sig_info), last_sig_info.si_status);
                        epoll_worker_exit();
                    }
                }
                uint32_t loc = hashtable_lookup(&ht_pid_to_ctx, &pid, sizeof(pid));
                if (0 == loc) {
                    LOGGER_ERROR("unhandled SIGCHLD detected, exiting...");
                    epoll_worker_exit();
                } else {
                    if (0 > queue_current_ctx()) {
                        LOGGER_ERROR("failed to queue current context, exiting...");
                        return epoll_worker_exit();
                    }
                    struct ribs_context *ctx = *(struct ribs_context **)hashtable_get_val(&ht_pid_to_ctx, loc);
                    hashtable_remove(&ht_pid_to_ctx, &pid, sizeof(pid));
                    ribs_swapcurcontext(ctx);
                }
            }
            epoll_worker_resume_events(sigfd);
        }
    }
}
Esempio n. 8
0
int mysql_dumper_dump(struct mysql_login_info *mysql_login_info, const char *outputdir, const char *dbname, const char *tablename, const char *query, size_t query_len, struct mysql_dumper_type *types) {
    MYSQL mysql;
    MYSQL_STMT *stmt = NULL;
    mysql_init(&mysql);
    my_bool b_flag = 1;
    if (0 != mysql_options(&mysql, MYSQL_OPT_RECONNECT, (const char *)&b_flag))
        return report_error(&mysql);

    if (NULL == mysql_real_connect(&mysql, mysql_login_info->host, mysql_login_info->user, mysql_login_info->pass, mysql_login_info->db, mysql_login_info->port, NULL, CLIENT_COMPRESS))
        return report_error(&mysql);

    b_flag = 0;
    if (0 != mysql_options(&mysql, MYSQL_REPORT_DATA_TRUNCATION, (const char *)&b_flag))
        return report_error(&mysql);

    stmt = mysql_stmt_init(&mysql);
    if (!stmt)
        return report_error(&mysql);

    if (0 != mysql_stmt_prepare(stmt, query, query_len))
        return report_stmt_error(&mysql, stmt);

    MYSQL_RES *rs = mysql_stmt_result_metadata(stmt);
    if (!rs)
        return report_stmt_error(&mysql, stmt);

    unsigned int n = mysql_num_fields(rs);
    MYSQL_FIELD *fields = mysql_fetch_fields(rs);
    int field_types[n];
    MYSQL_BIND bind[n];
    my_bool is_null[n];
    unsigned long length[n];
    my_bool error[n];
    memset(bind, 0, sizeof(MYSQL_BIND) * n);
    int null_terminate_str[n];
    memset(null_terminate_str, 0, sizeof(int) * n);

    struct file_writer ffields[n];
    struct file_writer vfields[2][n];

    struct vmbuf buf = VMBUF_INITIALIZER;
    vmbuf_init(&buf, 4096);
    vmbuf_sprintf(&buf, "%s/%s/%s/schema.txt", outputdir, dbname, tablename);
    mkdir_for_file_recursive(vmbuf_data(&buf));
    int fdschema = creat(vmbuf_data(&buf), 0644);
    struct vmfile ds_txt = VMFILE_INITIALIZER;
    vmbuf_reset(&buf);
    vmbuf_sprintf(&buf, "%s/%s/%s/ds.txt", outputdir, dbname, tablename);
    if (0 > vmfile_init(&ds_txt, vmbuf_data(&buf), 4096))
        return LOGGER_ERROR("failed to create: %s", vmbuf_data(&buf)), vmbuf_free(&buf), -1;
    vmfile_sprintf(&ds_txt, "DS_LOADER_BEGIN()\n");
    vmfile_sprintf(&ds_txt, "/*\n * DB: %s\n */\n", dbname);
    vmfile_sprintf(&ds_txt, "#undef DB_NAME\n#define DB_NAME %s\n", dbname);
    ssize_t len = 80 - strlen(tablename);
    if (len < 0) len = 4;
    char header[len];
    memset(header, '=', len);
    vmfile_sprintf(&ds_txt, "/* %.*s[ %s ]%.*s */\n", (int)len / 2, header, tablename, (int)(len - (len / 2)), header);
    vmfile_sprintf(&ds_txt, "#   undef TABLE_NAME\n#   define TABLE_NAME %s\n", tablename);
    /*
     * initialize output files
     */
    unsigned int i;
    for (i = 0; i < n; ++i) {
        file_writer_make(&ffields[i]);
        file_writer_make(&vfields[0][i]);
        file_writer_make(&vfields[1][i]);
    }

    struct hashtable ht_types = HASHTABLE_INITIALIZER;
    hashtable_init(&ht_types, 32);
    if (NULL != types) {
        struct mysql_dumper_type *t = types;
        for (; t->name; ++t) {
            /* storing ptr here which points to static str */
            hashtable_insert(&ht_types, t->name, strlen(t->name), t, sizeof(struct mysql_dumper_type));
        }
    }
    /*
     * parse meta data and construct bind array
     */
    int err = 0;
    for (i = 0; i < n; ++i) {
        field_types[i] = fields[i].type;
        bind[i].is_unsigned = IS_UNSIGNED(fields[i].flags);

        int64_t ds_type = -1;
        const char *ds_type_str = "VAR";
        /*
         * handle overrides
         */
        while (NULL != types) {
            uint32_t ofs = hashtable_lookup(&ht_types, fields[i].name, strlen(fields[i].name));
            if (!ofs) break;
            struct mysql_dumper_type *type = (struct mysql_dumper_type *)hashtable_get_val(&ht_types, ofs);
            null_terminate_str[i] = MYSQL_DUMPER_CSTR & type->flags;
            if (type->mysql_type)
                field_types[i] = type->mysql_type;
            bind[i].is_unsigned = (type->flags & MYSQL_DUMPER_UNSIGNED) > 0 ? 1 : 0;
            break;
        }
        vmbuf_reset(&buf);
        vmbuf_sprintf(&buf, "%s/%s/%s/%s", outputdir, dbname, tablename, fields[i].name);

        mkdir_for_file_recursive(vmbuf_data(&buf));
        if (is_var_length_field(field_types[i])) {
            size_t ofs = vmbuf_wlocpos(&buf);
            vmbuf_sprintf(&buf, ".ofs");
            if (0 > (err = file_writer_init(&vfields[0][i], vmbuf_data(&buf))))
                break;
            vmbuf_wlocset(&buf, ofs);
            vmbuf_sprintf(&buf, ".dat");
            if (0 > (err = file_writer_init(&vfields[1][i], vmbuf_data(&buf))))
                break;
        } else {
            ds_type = get_ds_type(field_types[i], bind[i].is_unsigned);
            const char *s = get_ds_type_str(ds_type);
            if (*s) ds_type_str = s;
            if (0 > (err = file_writer_init(&ffields[i], vmbuf_data(&buf))) ||
                0 > (err = file_writer_write(&ffields[i], &ds_type, sizeof(ds_type))))
                break;;
        }
        len = ribs_mysql_get_storage_size(field_types[i], fields[i].length);
        if (fdschema > 0)
            dprintf(fdschema, "%03d  name = %s, size=%zu, length=%lu, type=%s (%s), is_prikey=%d, ds_type=%s\n", i, fields[i].name, len, fields[i].length, ribs_mysql_get_type_name(field_types[i]), bind[i].is_unsigned ? "unsigned" : "signed", IS_PRI_KEY(fields[i].flags), ds_type_str);
        if (is_var_length_field(field_types[i])) {
            vmfile_sprintf(&ds_txt, "        DS_VAR_FIELD_LOADER(%s)\n", fields[i].name);
        } else {
            vmfile_sprintf(&ds_txt, "        DS_FIELD_LOADER(%s, %s)\n", ds_type_str, fields[i].name);
        }
        bind[i].buffer_type = field_types[i];
        bind[i].buffer_length = len;
        bind[i].buffer = malloc(len);
        bind[i].is_null = &is_null[i];
        bind[i].length = &length[i];
        bind[i].error = &error[i];
    }
    hashtable_free(&ht_types);
    mysql_free_result(rs);
    close(fdschema);
    //vmfile_sprintf(&ds_txt, "/*\n * TABLE END: %s\n */\n", tablename);
    vmfile_sprintf(&ds_txt, "DS_LOADER_END()\n");
    vmfile_close(&ds_txt);
    /*
     * execute & bind
     */
    if (0 != err ||
        0 != mysql_stmt_execute(stmt) ||
        0 != mysql_stmt_bind_result(stmt, bind)) {
        err = -1;
        report_stmt_error(&mysql, stmt);
        goto dumper_close_writer;
    }
    char zeros[4096];
    memset(zeros, 0, sizeof(zeros));
    int mysql_err = 0;
    size_t count = 0, num_rows_errors = 0;
    /*
     * write all rows to output files
     */
    while (0 == (mysql_err = mysql_stmt_fetch(stmt))) {
        int b = 0;
        for (i = 0; i < n && !b; ++i)
            b = b || error[i];
        if (b) {
            ++num_rows_errors;
            continue;
        }
        for (i = 0; i < n; ++i) {
            if (is_var_length_field(field_types[i])) {
                size_t ofs = file_writer_wlocpos(&vfields[1][i]);
                if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs))) ||
                    0 > (err = file_writer_write(&vfields[1][i], is_null[i] ? NULL : bind[i].buffer, is_null[i] ? 0 : length[i])))
                    goto dumper_error;
                if (null_terminate_str[i]) {
                    const char c = '\0';
                    if (0 > (err = file_writer_write(&vfields[1][i], &c, sizeof(c))))
                        goto dumper_error;
                }
            } else {
                if (0 > (err = file_writer_write(&ffields[i], is_null[i] ? zeros : bind[i].buffer, bind[i].buffer_length)))
                    goto dumper_error;
            }
        }
        ++count;
    }
    /* no dumper errors */
    goto dumper_ok;
 dumper_error:
    LOGGER_ERROR("failed to write data, aborting");
 dumper_ok:
    /* we are done with mysql, close it */
    mysql_stmt_close(stmt);
    mysql_close(&mysql);
    LOGGER_INFO("%s: %zu records, %zu skipped", tablename, count, num_rows_errors);
    /* check for mysql errors */
    if (mysql_err != MYSQL_NO_DATA) {
        LOGGER_ERROR("mysql_stmt_fetch returned an error (code=%d)\n", mysql_err);
        err = -1;
    }
 dumper_close_writer:
    /*
     * finalize & free memory
     */
    for (i = 0; i < n; ++i) {
        if (is_var_length_field(field_types[i])) {
            size_t ofs = file_writer_wlocpos(&vfields[1][i]);
            if (0 > (err = file_writer_write(&vfields[0][i], &ofs, sizeof(ofs))))
                LOGGER_ERROR("failed to write offset");
            file_writer_close(&vfields[0][i]);
            file_writer_close(&vfields[1][i]);
        } else {
            file_writer_close(&ffields[i]);
        }
        free(bind[i].buffer);
    }
    vmbuf_free(&buf);
    return err;
}