void conn_cleanup(struct conn *c) { ASSERT(c != NULL); if (c->item != NULL) { item_remove(c->item); c->item = NULL; } while (c->ileft > 0) { item_remove(*(c->icurr)); c->ileft--; c->icurr++; } while (c->sleft > 0) { cache_free(c->thread->suffix_cache, *(c->scurr)); c->sleft--; c->scurr++; } if (c->write_and_free != NULL) { mc_free(c->write_and_free); } }
int process_update_command ( char *key, size_t nkey, uint32_t flags, char *value, size_t nbytes, int32_t exptime_int, int comm ) { item *it; enum store_item_type ret; int32_t vlen = ( int32_t ) nbytes; if ( exptime_int < 0 ) exptime_int = REALTIME_MAXDELTA + 1; vlen += 2; if ( vlen < 0 || vlen - 2 < 0 ) { return false; } it = item_alloc (key, nkey, flags, realtime (exptime_int), value, vlen); if ( it == 0 ) { if ( comm == NREAD_SET ) { it = item_get (key, nkey); if ( it ) { item_unlink (it); item_remove (it); } } return false; } ret = store_item (it, comm); item_remove (it); if ( ret != STORED ) { return false; } return true; }
bool cproxy_binary_ignore_reply(conn *c, protocol_binary_response_header *header, item *it) { if (c->noreply && OPAQUE_IGNORE_REPLY == ntohl(header->response.opaque)) { /* Handle when the client sent an ascii noreply command, */ /* and we now need to eat the binary error responses. */ /* So, drop the current response (should be an error response) */ /* and go to read the next response message. */ if (settings.verbose > 2) { moxi_log_write("<%d cproxy_process_a2b_downstream_response OPAQUE_IGNORE_REPLY, " "cmd: %x, status: %x, ignoring reply\n", c->sfd, header->response.opcode, header->response.status); } conn_set_state(c, conn_new_cmd); if (it != NULL) { item_remove(it); } return true; } return false; }
/* We get here after reading the value in a VALUE reply. * The item is ready in c->item. */ void cproxy_process_a2a_downstream_nread(conn *c) { assert(c != NULL); if (settings.verbose > 1) fprintf(stderr, "<%d cproxy_process_a2a_downstream_nread %d %d\n", c->sfd, c->ileft, c->isize); downstream *d = c->extra; assert(d != NULL); item *it = c->item; assert(it != NULL); // Clear c->item because we either move it to the upstream or // item_remove() it on error. // c->item = NULL; conn_set_state(c, conn_new_cmd); // pthread_mutex_lock(&c->thread->stats.mutex); // c->thread->stats.slab_stats[it->slabs_clsid].set_cmds++; // pthread_mutex_unlock(&c->thread->stats.mutex); multiget_ascii_downstream_response(d, it); item_remove(it); }
void conn_close(conn *c) { /* delete the event, the socket and the conn */ event_del(&c->event); close(c->sfd); free(c->rbuf); free(c->wbuf); if (c->item) { free(c->item); } if (c->ileft) { for (; c->ileft > 0; c->ileft--,c->icurr++) { item_remove(*(c->icurr)); } } free(c->ilist); if (c->write_and_free) { free(c->write_and_free); } /* if we have enough space in the free connections array, put the structure there */ if (freecurr < freetotal) { freeconns[freecurr++] = c; } else { free(c); } stats.curr_conns--; /* if (stats.curr_conns % 10 == 0) printf("conns: %d\n", stats.curr_conns); */ return; }
/* We get here after reading the value in a VALUE reply. * The item is ready in c->item. */ void cproxy_process_a2a_downstream_nread(conn *c) { cb_assert(c != NULL); if (settings.verbose > 1) { moxi_log_write("<%d cproxy_process_a2a_downstream_nread %d %d\n", c->sfd, c->ileft, c->isize); } downstream *d = c->extra; cb_assert(d != NULL); item *it = c->item; cb_assert(it != NULL); /* Clear c->item because we either move it to the upstream or */ /* item_remove() it on error. */ c->item = NULL; conn_set_state(c, conn_new_cmd); /* pthread_mutex_lock(&c->thread->stats.mutex); */ /* c->thread->stats.slab_stats[it->slabs_clsid].set_cmds++; */ /* pthread_mutex_unlock(&c->thread->stats.mutex); */ multiget_ascii_downstream_response(d, it); item_remove(it); }
void actor_item(Actor * self, Item * it) { if (it->kind == IT_CHEESE) { if (self->kind->enemy) { return; } self->max_hp += 5; self->hp = self->max_hp; item_remove(it); } }
void cproxy_binary_uncork_cmds(downstream *d, conn *uc) { cb_assert(d != NULL); cb_assert(uc != NULL); if (settings.verbose > 2) { moxi_log_write("%d: cproxy_binary_uncork_cmds\n", uc->sfd); } int n = 0; while (uc->corked != NULL) { bin_cmd *next = uc->corked->next; item *it = uc->corked->request_item; if (it != NULL) { b2b_forward_item(uc, d, it); n++; } if (uc->corked->request_item != NULL) { item_remove(uc->corked->request_item); } if (uc->corked->response_item != NULL) { item_remove(uc->corked->response_item); } free(uc->corked); uc->corked = next; } if (settings.verbose > 2) { moxi_log_write("%d: cproxy_binary_uncork_cmds, uncorked %d\n", uc->sfd, n); } }
void HTMLButcherAssortedFilesDialog::do_remove(unsigned long id) { wxString remmsg=_("Are you sure you want to remove this file?"); wxMessageDialog d(this, remmsg, _("Remove file"), wxYES_NO | wxYES_DEFAULT | wxICON_QUESTION); if (d.ShowModal() != wxID_YES) return; ButcherProjectBaseAutoUpdate upd(GetProject()); if (!GetProject()->AssortedFiles().Delete(id)) { butil_errordialog(_("This file cannot be deleted"), this); return; } item_remove(id); }
int process_touch_command ( char *key, size_t nkey, int32_t exptime_int ) { item *it; it = item_touch (key, nkey, realtime (exptime_int)); if ( it ) { item_update (it); item_remove (it); return true; } else { //item_remove(it); return false; } }
int process_delete_command ( char *key, size_t nkey ) { item *it; it = item_get (key, nkey); if ( it ) { item_unlink (it); item_remove (it); return true; } else { //item_remove(it); return false; } }
/* * We get here after reading the value in update commands. The command * is stored in c->req_type, and the item is ready in c->item. */ void asc_complete_nread(struct conn *c) { item_store_result_t ret; struct item *it; char *end; it = c->item; end = item_data(it) + it->nbyte; if (!strcrlf(end)) { log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for " "req of type %d with missing crlf", c->sd, c->req_type); asc_write_client_error(c); } else { ret = item_store(it, c->req_type, c); switch (ret) { case STORED: asc_write_stored(c); break; case EXISTS: asc_write_exists(c); break; case NOT_FOUND: asc_write_not_found(c); break; case NOT_STORED: asc_write_not_stored(c); break; default: log_warn("server error on c %d for req of type %d with unknown " "store result %d", c->sd, c->req_type, ret); asc_write_server_error(c); break; } } item_remove(c->item); c->item = NULL; }
void conn_close(conn *c) { /* delete the event, the socket and the conn */ event_del(&c->event); if (settings.verbose > 1) fprintf(stderr, "<%d connection closed.\n", c->sfd); close(c->sfd); if (c->item) { item_free(c->item); } if (c->ileft) { for (; c->ileft > 0; c->ileft--,c->icurr++) { item_remove(*(c->icurr)); } } if (c->write_and_free) { free(c->write_and_free); } /* if we have enough space in the free connections array, put the structure there */ if (freecurr < freetotal) { freeconns[freecurr++] = c; } else { /* try to enlarge free connections array */ conn **new_freeconns = realloc(freeconns, sizeof(conn *)*freetotal*2); if (new_freeconns) { freetotal *= 2; freeconns = new_freeconns; freeconns[freecurr++] = c; } else { free(c->rbuf); free(c->wbuf); free(c->ilist); free(c); } } stats.curr_conns--; return; }
void * thrash(void *arg) { ASSERT(arg); ThreadTest *tt = (ThreadTest*) arg; Item **head = & tt->head; Mutex *mutex = & tt->mutex; Item *item = 0; int num = 100; Item items[num]; // Initialise each item for (int i = 0; i < num; i++) { init_item(& items[i], i); } // create a randomised table of indexes int block[num]; rand_array(block, num); // add our items randomly to sorted list for (int i = 0; i < num; i++) { int idx = block[i]; item = & items[idx]; item_add_sorted(head, item, mutex); } // Remove the items for (int i = 0; i < num; i++) { item = & items[i]; bool found = item_remove(head, item, mutex); EXPECT_TRUE(found); // check it is the correct item EXPECT_EQ(& items[i], item); // check the item is correct item_validate(item, i); } return 0; }
int process_get_command ( char *key, size_t nkey, char *dst, int *len ) { item *it; char *src = NULL; it = item_get (key, nkey); if ( it ) { item_update (it); src = ITEM_data (it); *len = it->nbytes - 2; memcpy (dst, src, it->nbytes); item_remove (it); return true; } else { //item_remove(it); return false; } }
// complete the response to a get request. void finish_get_command(conn *c) { item *it; int i; // setup all items for writing out. for (i = 0; i < c->ileft; i++) { it = *(c->ilist + i); if (it) { // Construct the response. Each hit adds three elements to the // outgoing data list: // "VALUE <key> <flags> <data_length>\r\n" // "<data>\r\n" // The <data> element is stored on the connection item list, not on // the iov list. if (!conn_add_iov(c, "VALUE ", 6) || !conn_add_iov(c, ITEM_key(it), it->nkey) || !conn_add_iov(c, ITEM_suffix(it), it->nsuffix + it->nbytes)) { item_remove(it); error_response(c, "SERVER_ERROR out of memory writing get response"); return; } if (config.verbose > 1) { fprintf(stderr, ">%d sending key %s\n", c->sfd, ITEM_key(it)); } } else { fprintf(stderr, "ERROR corrupted ilist!\n"); exit(1); } } if (config.verbose > 1) { fprintf(stderr, ">%d END\n", c->sfd); } if (!conn_add_iov(c, "END\r\n", 5) != 0) { error_response(c, "SERVER_ERROR out of memory writing get response"); } else { conn_set_state(c, conn_mwrite); } }
void HTMLButcherCSSFilesDialog::do_remove(unsigned long id) { // get number of views using this mask ButcherProjectEventNotify en(GetProject(), ButcherProjectEvent::BPE_CSSFILEDELETED, id, 0, false); GetProject()->ExecuteEventNotify(en); wxString remmsg=_("Are you sure you want to remove this CSS?"); if (en.GetRemoveCount()>0||en.GetChangeCount()>0) remmsg=wxString::Format(_("Removing this CSS will affect %d item(s). Are you sure?"), en.GetRemoveCount()+en.GetChangeCount()); wxMessageDialog d(this, remmsg, _("Remove CSS"), wxYES_NO | wxYES_DEFAULT | wxICON_QUESTION); if (d.ShowModal() != wxID_YES) return; ButcherProjectBaseAutoUpdate upd(GetProject()); if (!GetProject()->CSSFiles().Delete(id)) { butil_errordialog(_("This CSS cannot be deleted"), this); return; } item_remove(id); }
static void asc_process_delete(struct conn *c, struct token *token, int ntoken) { char *key; /* key to be deleted */ size_t nkey; /* # key bytes */ struct item *it; /* item for this key */ asc_set_noreply_maybe(c, token, ntoken); if (!asc_ntoken_valid(c, ntoken)) { log_hexdump(LOG_INFO, c->req, c->req_len, "client error on c %d for " "req of type %d with %d invalid tokens", c->sd, c->req_type, ntoken); asc_write_client_error(c); return; } key = token[TOKEN_KEY].val; nkey = token[TOKEN_KEY].len; if (nkey > KEY_MAX_LEN) { log_debug(LOG_INFO, "client error on c %d for req of type %d and %d " "length key", c->sd, c->req_type, nkey); asc_write_client_error(c); return; } it = item_get(key, nkey); if (it != NULL) { stats_slab_incr(it->id, delete_hit); item_unlink(it); item_remove(it); asc_write_deleted(c); } else { stats_thread_incr(delete_miss); asc_write_not_found(c); } }
void HTMLButcherMasksDialog::do_remove(unsigned long id) { // get number of views using this mask ButcherProjectEventNotify en(GetProject(), ButcherProjectEvent::BPE_MASKDELETED, id, 0, false); GetProject()->ExecuteEventNotify(en); wxString remmsg=_("Are you sure you want to remove this mask?"); if (en.GetRemoveCount()>0) remmsg=wxString::Format(_("Removing this mask will remove %d view(s). Are you sure?"), en.GetRemoveCount()); wxMessageDialog d(this, remmsg, _("Remove mask"), wxYES_NO | wxYES_DEFAULT | wxICON_QUESTION); if (d.ShowModal() != wxID_YES) return; ButcherProjectBaseAutoUpdate upd(GetProject()); if (!GetProject()->Masks().Delete(id)) { butil_errordialog(_("This mask cannot be deleted"), this); return; } item_remove(id); EnableAdd(!GetProject()->Masks().IsFull()); }
void delete_handler(int fd, short which, void *arg) { struct timeval t; static int initialized = 0; if (initialized) { /* some versions of libevent don't like deleting events that don't exist, so only delete once we know this event has been added. */ evtimer_del(&deleteevent); } else { initialized = 1; } evtimer_set(&deleteevent, delete_handler, 0); t.tv_sec = 5; t.tv_usec=0; evtimer_add(&deleteevent, &t); { int i, j=0; time_t now = time(0); for (i=0; i<delcurr; i++) { item *it = todelete[i]; if (it->exptime < now) { assert(it->refcount > 0); it->it_flags &= ~ITEM_DELETED; item_unlink(it); item_remove(it); } else { todelete[j++] = it; } } delcurr = j; } return; }
void cproxy_process_a2a_downstream(conn *c, char *line) { assert(c != NULL); assert(c->next == NULL); assert(c->extra != NULL); assert(c->cmd == -1); assert(c->item == NULL); assert(line != NULL); assert(line == c->rcurr); assert(IS_ASCII(c->protocol)); assert(IS_PROXY(c->protocol)); if (settings.verbose > 1) fprintf(stderr, "<%d cproxy_process_a2a_downstream %s\n", c->sfd, line); downstream *d = c->extra; assert(d != NULL); assert(d->ptd != NULL); assert(d->ptd->proxy != NULL); if (strncmp(line, "VALUE ", 6) == 0) { token_t tokens[MAX_TOKENS]; size_t ntokens; unsigned int flags; int clen = 0; int vlen; uint64_t cas = CPROXY_NOT_CAS; ntokens = scan_tokens(line, tokens, MAX_TOKENS, &clen); if (ntokens >= 5 && // Accounts for extra termimation token. ntokens <= 6 && tokens[KEY_TOKEN].length <= KEY_MAX_LENGTH && safe_strtoul(tokens[2].value, (uint32_t *) &flags) && safe_strtoul(tokens[3].value, (uint32_t *) &vlen)) { char *key = tokens[KEY_TOKEN].value; size_t nkey = tokens[KEY_TOKEN].length; item *it = item_alloc(key, nkey, flags, 0, vlen + 2); if (it != NULL) { if (ntokens == 5 || safe_strtoull(tokens[4].value, &cas)) { ITEM_set_cas(it, cas); c->item = it; c->ritem = ITEM_data(it); c->rlbytes = it->nbytes; c->cmd = -1; conn_set_state(c, conn_nread); return; // Success. } else { if (settings.verbose > 1) fprintf(stderr, "cproxy could not parse cas\n"); } } else { if (settings.verbose > 1) fprintf(stderr, "cproxy could not item_alloc size %u\n", vlen + 2); } if (it != NULL) item_remove(it); it = NULL; c->sbytes = vlen + 2; // Number of bytes to swallow. conn_set_state(c, conn_swallow); // Note, eventually, we'll see an END later. } else { // We don't know how much to swallow, so close the downstream. // The conn_closing should release the downstream, // which should write a suffix/error to the upstream. // conn_set_state(c, conn_closing); } } else if (strncmp(line, "END", 3) == 0) { conn_set_state(c, conn_pause); } else if (strncmp(line, "OK", 2) == 0) { conn_set_state(c, conn_pause); // TODO: Handle flush_all's expiration parameter against // the front_cache. // // TODO: We flush the front_cache too often, inefficiently // on every downstream flush_all OK response, rather than // on just the last flush_all OK response. // conn *uc = d->upstream_conn; if (uc != NULL && uc->cmd_curr == PROTOCOL_BINARY_CMD_FLUSH) { mcache_flush_all(&d->ptd->proxy->front_cache, 0); } } else if (strncmp(line, "STAT ", 5) == 0 || strncmp(line, "ITEM ", 5) == 0 || strncmp(line, "PREFIX ", 7) == 0) { assert(d->merger != NULL); conn *uc = d->upstream_conn; if (uc != NULL) { assert(uc->next == NULL); if (protocol_stats_merge_line(d->merger, line) == false) { // Forward the line as-is if we couldn't merge it. // int nline = strlen(line); item *it = item_alloc("s", 1, 0, 0, nline + 2); if (it != NULL) { strncpy(ITEM_data(it), line, nline); strncpy(ITEM_data(it) + nline, "\r\n", 2); if (add_conn_item(uc, it)) { add_iov(uc, ITEM_data(it), nline + 2); it = NULL; } if (it != NULL) item_remove(it); } } } conn_set_state(c, conn_new_cmd); } else { conn_set_state(c, conn_pause); // The upstream conn might be NULL when closed already // or while handling a noreply. // conn *uc = d->upstream_conn; if (uc != NULL) { assert(uc->next == NULL); out_string(uc, line); if (!update_event(uc, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 1) fprintf(stderr, "Can't update upstream write event\n"); d->ptd->stats.stats.err_oom++; cproxy_close_conn(uc); } cproxy_del_front_cache_key_ascii_response(d, line, uc->cmd_start); } } }
static inline void asc_process_read(struct conn *c, struct token *token, int ntoken) { rstatus_t status; char *key; size_t nkey; unsigned valid_key_iter = 0; struct item *it; struct token *key_token; bool return_cas; if (!asc_ntoken_valid(c, ntoken)) { log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for " "req of type %d with %d invalid tokens", c->sd, c->req_type, ntoken); asc_write_client_error(c); return; } return_cas = (c->req_type == REQ_GETS) ? true : false; key_token = &token[TOKEN_KEY]; do { while (key_token->len != 0) { key = key_token->val; nkey = key_token->len; if (nkey > KEY_MAX_LEN) { log_debug(LOG_NOTICE, "client error on c %d for req of type %d " "and %d length key", c->sd, c->req_type, nkey); asc_write_client_error(c); return; } if (return_cas) { stats_thread_incr(gets); } else { stats_thread_incr(get); } it = item_get(key, nkey); if (it != NULL) { /* item found */ if (return_cas) { stats_slab_incr(it->id, gets_hit); } else { stats_slab_incr(it->id, get_hit); } if (valid_key_iter >= c->isize) { struct item **new_list; new_list = mc_realloc(c->ilist, sizeof(struct item *) * c->isize * 2); if (new_list != NULL) { c->isize *= 2; c->ilist = new_list; } else { item_remove(it); break; } } status = asc_respond_get(c, valid_key_iter, it, return_cas); if (status != MC_OK) { log_debug(LOG_NOTICE, "client error on c %d for req of type " "%d with %d tokens", c->sd, c->req_type, ntoken); stats_thread_incr(cmd_error); item_remove(it); break; } log_debug(LOG_VVERB, ">%d sending key %.*s", c->sd, it->nkey, item_key(it)); item_touch(it); *(c->ilist + valid_key_iter) = it; valid_key_iter++; } else { /* item not found */ if (return_cas) { stats_thread_incr(gets_miss); } else { stats_thread_incr(get_miss); } klog_write(c->peer, c->req_type, key, nkey, 1, 0); } key_token++; } /* * If the command string hasn't been fully processed, get the next set * of token. */ if (key_token->val != NULL) { ntoken = asc_tokenize(key_token->val, token, TOKEN_MAX); /* ntoken is unused */ key_token = token; } } while (key_token->val != NULL); c->icurr = c->ilist; c->ileft = valid_key_iter; if (return_cas) { c->scurr = c->slist; c->sleft = valid_key_iter; } log_debug(LOG_VVERB, ">%d END", c->sd); /* * If the loop was terminated because of out-of-memory, it is not * reliable to add END\r\n to the buffer, because it might not end * in \r\n. So we send SERVER_ERROR instead. */ if (key_token->val != NULL || conn_add_iov(c, "END\r\n", 5) != MC_OK || (c->udp && conn_build_udp_headers(c) != MC_OK)) { log_warn("server error on c %d for req of type %d with enomem", c->sd, c->req_type); asc_write_server_error(c); } else { conn_set_state(c, CONN_MWRITE); c->msg_curr = 0; } }
void drive_machine(conn *c) { int exit = 0; int sfd, flags = 1; socklen_t addrlen; struct sockaddr addr; conn *newc; int res; while (!exit) { /* printf("state %d\n", c->state);*/ switch(c->state) { case conn_listening: addrlen = sizeof(addr); if ((sfd = accept(c->sfd, &addr, &addrlen)) == -1) { if (errno == EAGAIN || errno == EWOULDBLOCK) { exit = 1; break; } else { perror("accept()"); } break; } if ((flags = fcntl(sfd, F_GETFL, 0)) < 0 || fcntl(sfd, F_SETFL, flags | O_NONBLOCK) < 0) { perror("setting O_NONBLOCK"); close(sfd); break; } newc = conn_new(sfd, conn_read, EV_READ | EV_PERSIST); if (!newc) { if (settings.verbose > 0) fprintf(stderr, "couldn't create new connection\n"); close(sfd); break; } break; case conn_read: if (try_read_command(c)) { continue; } if (try_read_network(c)) { continue; } /* we have no command line and no data to read from network */ if (!update_event(c, EV_READ | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; } exit = 1; break; case conn_nread: /* we are reading rlbytes into rcurr; */ if (c->rlbytes == 0) { complete_nread(c); break; } /* first check if we have leftovers in the conn_read buffer */ if (c->rbytes > 0) { int tocopy = c->rbytes > c->rlbytes ? c->rlbytes : c->rbytes; memcpy(c->rcurr, c->rbuf, tocopy); c->rcurr += tocopy; c->rlbytes -= tocopy; if (c->rbytes > tocopy) { memmove(c->rbuf, c->rbuf+tocopy, c->rbytes - tocopy); } c->rbytes -= tocopy; break; } /* now try reading from the socket */ res = read(c->sfd, c->rcurr, c->rlbytes); if (res > 0) { stats.bytes_read += res; c->rcurr += res; c->rlbytes -= res; break; } if (res == 0) { /* end of stream */ c->state = conn_closing; break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_READ | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; } exit = 1; break; } /* otherwise we have a real error, on which we close the connection */ if (settings.verbose > 0) fprintf(stderr, "Failed to read, and not due to blocking\n"); c->state = conn_closing; break; case conn_swallow: /* we are reading sbytes and throwing them away */ if (c->sbytes == 0) { c->state = conn_read; break; } /* first check if we have leftovers in the conn_read buffer */ if (c->rbytes > 0) { int tocopy = c->rbytes > c->sbytes ? c->sbytes : c->rbytes; c->sbytes -= tocopy; if (c->rbytes > tocopy) { memmove(c->rbuf, c->rbuf+tocopy, c->rbytes - tocopy); } c->rbytes -= tocopy; break; } /* now try reading from the socket */ res = read(c->sfd, c->rbuf, c->rsize > c->sbytes ? c->sbytes : c->rsize); if (res > 0) { stats.bytes_read += res; c->sbytes -= res; break; } if (res == 0) { /* end of stream */ c->state = conn_closing; break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_READ | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; } exit = 1; break; } /* otherwise we have a real error, on which we close the connection */ if (settings.verbose > 0) fprintf(stderr, "Failed to read, and not due to blocking\n"); c->state = conn_closing; break; case conn_write: /* we are writing wbytes bytes starting from wcurr */ if (c->wbytes == 0) { if (c->write_and_free) { free(c->write_and_free); c->write_and_free = 0; } c->state = c->write_and_go; if (c->state == conn_read) set_cork(c, 0); break; } res = write(c->sfd, c->wcurr, c->wbytes); if (res > 0) { stats.bytes_written += res; c->wcurr += res; c->wbytes -= res; break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; } exit = 1; break; } /* if res==0 or res==-1 and error is not EAGAIN or EWOULDBLOCK, we have a real error, on which we close the connection */ if (settings.verbose > 0) fprintf(stderr, "Failed to write, and not due to blocking\n"); c->state = conn_closing; break; case conn_mwrite: /* * we're writing ibytes bytes from iptr. iptr alternates between * ibuf, where we build a string "VALUE...", and ITEM_data(it) for the * current item. When we finish a chunk, we choose the next one using * ipart, which has the following semantics: 0 - start the loop, 1 - * we finished ibuf, go to current ITEM_data(it); 2 - we finished ITEM_data(it), * move to the next item and build its ibuf; 3 - we finished all items, * write "END". */ if (c->ibytes > 0) { res = write(c->sfd, c->iptr, c->ibytes); if (res > 0) { stats.bytes_written += res; c->iptr += res; c->ibytes -= res; break; } if (res == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { if (!update_event(c, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 0) fprintf(stderr, "Couldn't update event\n"); c->state = conn_closing; break; } exit = 1; break; } /* if res==0 or res==-1 and error is not EAGAIN or EWOULDBLOCK, we have a real error, on which we close the connection */ if (settings.verbose > 0) fprintf(stderr, "Failed to write, and not due to blocking\n"); c->state = conn_closing; break; } else { item *it; /* we finished a chunk, decide what to do next */ switch (c->ipart) { case 1: it = *(c->icurr); assert((it->it_flags & ITEM_SLABBED) == 0); c->iptr = ITEM_data(it); c->ibytes = it->nbytes; c->ipart = 2; break; case 2: it = *(c->icurr); item_remove(it); c->ileft--; if (c->ileft <= 0) { c->ipart = 3; break; } else { c->icurr++; } /* FALL THROUGH */ case 0: it = *(c->icurr); assert((it->it_flags & ITEM_SLABBED) == 0); c->ibytes = sprintf(c->ibuf, "VALUE %s %u %u\r\n", ITEM_key(it), it->flags, it->nbytes - 2); if (settings.verbose > 1) fprintf(stderr, ">%d sending key %s\n", c->sfd, ITEM_key(it)); c->iptr = c->ibuf; c->ipart = 1; break; case 3: out_string(c, "END"); break; } } break; case conn_closing: conn_close(c); exit = 1; break; } } return; }
/* We reach here after nread'ing a header+body into an item. */ void cproxy_process_b2b_downstream_nread(conn *c) { conn *uc; item *it; downstream *d; protocol_binary_response_header *header; int extlen; int keylen; uint32_t bodylen; int status; int opcode; cb_assert(c != NULL); cb_assert(c->cmd >= 0); cb_assert(c->next == NULL); cb_assert(c->cmd_start == NULL); cb_assert(IS_BINARY(c->protocol)); cb_assert(IS_PROXY(c->protocol)); header = (protocol_binary_response_header *) &c->binary_header; extlen = header->response.extlen; keylen = header->response.keylen; bodylen = header->response.bodylen; status = ntohs(header->response.status); opcode = header->response.opcode; if (settings.verbose > 2) { moxi_log_write("<%d cproxy_process_b2b_downstream_nread %x %x %d %d %u %d %x\n", c->sfd, c->cmd, opcode, extlen, keylen, bodylen, c->noreply, status); } d = c->extra; cb_assert(d != NULL); cb_assert(d->ptd != NULL); cb_assert(d->ptd->proxy != NULL); /* TODO: Need to handle quiet binary command error response, */ /* in the right order. */ /* TODO: Need to handle not-my-vbucket error during a quiet cmd. */ uc = d->upstream_conn; it = c->item; /* Clear c->item because we either move it to the upstream or */ /* item_remove() it on error. */ c->item = NULL; cb_assert(it != NULL); cb_assert(it->refcount == 1); if (cproxy_binary_ignore_reply(c, header, it)) { return; } if (c->noreply) { conn_set_state(c, conn_new_cmd); } else { conn_set_state(c, conn_pause); if (opcode == PROTOCOL_BINARY_CMD_NOOP || opcode == PROTOCOL_BINARY_CMD_FLUSH) { goto done; } if (opcode == PROTOCOL_BINARY_CMD_STAT) { if (status == PROTOCOL_BINARY_RESPONSE_SUCCESS) { if (keylen > 0) { if (d->merger != NULL) { char *key = (ITEM_data(it)) + sizeof(*header) + extlen; char *val = key + keylen; protocol_stats_merge_name_val(d->merger, "STAT", 4, key, keylen, val, bodylen - keylen - extlen); } conn_set_state(c, conn_new_cmd); /* Get next STATS response. */ } } goto done; } /* If the client is still there, we should handle */ /* a not-my-vbucket error with a possible retry. */ if (uc != NULL && status == PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET) { int max_retries; protocol_binary_request_header *req; int vbucket; int sindex; if (settings.verbose > 2) { moxi_log_write("<%d cproxy_process_b2b_downstream_nread not-my-vbucket, " "cmd: %x %d\n", c->sfd, header->response.opcode, uc->item != NULL); } cb_assert(uc->item != NULL); req = (protocol_binary_request_header *)ITEM_data((item*)uc->item); vbucket = ntohs(req->request.reserved); sindex = downstream_conn_index(d, c); if (settings.verbose > 2) { moxi_log_write("<%d cproxy_process_b2b_downstream_nread not-my-vbucket, " "cmd: %x not multi-key get, sindex %d, vbucket %d, retries %d\n", c->sfd, header->response.opcode, sindex, vbucket, uc->cmd_retries); } mcs_server_invalid_vbucket(&d->mst, sindex, vbucket); /* As long as the upstream is still open and we haven't */ /* retried too many times already. */ max_retries = cproxy_max_retries(d); if (uc->cmd_retries < max_retries) { uc->cmd_retries++; d->upstream_retry++; d->ptd->stats.stats.tot_retry_vbucket++; goto done; } if (settings.verbose > 2) { moxi_log_write("%d: cproxy_process_b2b_downstream_nread not-my-vbucket, " "cmd: %x skipping retry %d >= %d\n", c->sfd, header->response.opcode, uc->cmd_retries, max_retries); } } } /* Write the response to the upstream connection. */ if (uc != NULL) { if (settings.verbose > 2) { moxi_log_write("<%d cproxy_process_b2b_downstream_nread got %u\n", c->sfd, it->nbytes); cproxy_dump_header(c->sfd, ITEM_data(it)); } if (add_conn_item(uc, it) == true) { it->refcount++; if (add_iov(uc, ITEM_data(it), it->nbytes) == 0) { /* If we got a quiet response, however, don't change the */ /* upstream connection's state (should be in paused state), */ /* as we expect the downstream server to provide a */ /* verbal/non-quiet response that moves the downstream */ /* conn through the conn_pause countdown codepath. */ if (c->noreply == false) { cproxy_update_event_write(d, uc); conn_set_state(uc, conn_mwrite); } goto done; } } d->ptd->stats.stats.err_oom++; cproxy_close_conn(uc); } done: if (it != NULL) { item_remove(it); } }
static void item_dec_ref(void *it) { item *i = it; if (i != NULL) { item_remove(i); } }
/* Used for broadcast commands, like no-op, flush_all or stats. */ bool cproxy_broadcast_b2b_downstream(downstream *d, conn *uc) { int nwrite = 0; int nconns; int i; cb_assert(d != NULL); cb_assert(d->ptd != NULL); cb_assert(d->ptd->proxy != NULL); cb_assert(d->downstream_conns != NULL); cb_assert(uc != NULL); cb_assert(uc->next == NULL); cb_assert(uc->noreply == false); nconns = mcs_server_count(&d->mst); for (i = 0; i < nconns; i++) { conn *c = d->downstream_conns[i]; if (c != NULL && c != NULL_CONN && b2b_forward_item_vbucket(uc, d, uc->item, c, -1) == true) { nwrite++; } } if (settings.verbose > 2) { moxi_log_write("%d: b2b broadcast nwrite %d out of %d\n", uc->sfd, nwrite, nconns); } if (nwrite > 0) { /* TODO: Handle binary 'stats reset' sub-command. */ item *it; if (uc->cmd == PROTOCOL_BINARY_CMD_STAT && d->merger == NULL) { d->merger = genhash_init(128, skeyhash_ops); } it = item_alloc("h", 1, 0, 0, sizeof(protocol_binary_response_header)); if (it != NULL) { protocol_binary_response_header *header = (protocol_binary_response_header *) ITEM_data(it); memset(ITEM_data(it), 0, it->nbytes); header->response.magic = (uint8_t) PROTOCOL_BINARY_RES; header->response.opcode = uc->binary_header.request.opcode; header->response.opaque = uc->opaque; if (add_conn_item(uc, it)) { d->upstream_suffix = ITEM_data(it); d->upstream_suffix_len = it->nbytes; d->upstream_status = PROTOCOL_BINARY_RESPONSE_SUCCESS; d->target_host_ident = NULL; if (settings.verbose > 2) { moxi_log_write("%d: b2b broadcast upstream_suffix", uc->sfd); cproxy_dump_header(uc->sfd, ITEM_data(it)); } /* TODO: Handle FLUSHQ (quiet binary flush_all). */ d->downstream_used_start = nwrite; d->downstream_used = nwrite; cproxy_start_downstream_timeout(d, NULL); return true; } item_remove(it); } } return false; }
bool multiget_ascii_downstream(downstream *d, conn *uc, int (*emit_start)(conn *c, char *cmd, int cmd_len), int (*emit_skey)(conn *c, char *skey, int skey_len), int (*emit_end)(conn *c), mcache *front_cache) { assert(d != NULL); assert(d->downstream_conns != NULL); assert(d->multiget == NULL); assert(uc != NULL); assert(uc->noreply == false); proxy_td *ptd = d->ptd; assert(ptd != NULL); proxy_stats_cmd *psc_get = &ptd->stats.stats_cmd[STATS_CMD_TYPE_REGULAR][STATS_CMD_GET]; proxy_stats_cmd *psc_get_key = &ptd->stats.stats_cmd[STATS_CMD_TYPE_REGULAR][STATS_CMD_GET_KEY]; int nwrite = 0; int nconns = mcs_server_count(&d->mst); for (int i = 0; i < nconns; i++) { if (d->downstream_conns[i] != NULL && cproxy_prep_conn_for_write(d->downstream_conns[i]) == false) { d->ptd->stats.stats.err_downstream_write_prep++; cproxy_close_conn(d->downstream_conns[i]); return false; } } if (uc->next != NULL) { // More than one upstream conn, so we need a hashtable // to track keys for de-deplication. // d->multiget = genhash_init(128, skeyhash_ops); if (settings.verbose > 1) { fprintf(stderr, "cproxy multiget hash table new\n"); } } // Snapshot the volatile only once. // uint32_t msec_current_time_snapshot = msec_current_time; int uc_num = 0; conn *uc_cur = uc; while (uc_cur != NULL) { assert(uc_cur->cmd == -1); assert(uc_cur->item == NULL); assert(uc_cur->state == conn_pause); assert(IS_ASCII(uc_cur->protocol)); assert(IS_PROXY(uc_cur->protocol)); char *command = uc_cur->cmd_start; assert(command != NULL); char *space = strchr(command, ' '); assert(space > command); int cmd_len = space - command; assert(cmd_len == 3 || cmd_len == 4); // Either get or gets. int cas_emit = (command[3] == 's'); if (settings.verbose > 1) { fprintf(stderr, "forward multiget %s (%d %d)\n", command, cmd_len, uc_num); } while (space != NULL) { char *key = space + 1; char *next_space = strchr(key, ' '); int key_len; if (next_space != NULL) { key_len = next_space - key; } else { key_len = strlen(key); // We've reached the last key. // psc_get->read_bytes += (key - command + key_len); } // This key_len check helps skip consecutive spaces. // if (key_len > 0) { ptd->stats.stats.tot_multiget_keys++; psc_get_key->seen++; psc_get_key->read_bytes += key_len; // Update key-based statistics. // bool do_key_stats = matcher_check(&ptd->key_stats_matcher, key, key_len, true) == true && matcher_check(&ptd->key_stats_unmatcher, key, key_len, false) == false; if (do_key_stats) { touch_key_stats(ptd, key, key_len, msec_current_time_snapshot, STATS_CMD_TYPE_REGULAR, STATS_CMD_GET_KEY, 1, 0, 0, key_len, 0); } // Handle a front cache hit by queuing response. // // Note, front cache stats are part of mcache. // if (!cas_emit) { item *it = mcache_get(front_cache, key, key_len, msec_current_time_snapshot); if (it != NULL) { assert(it->nkey == key_len); assert(strncmp(ITEM_key(it), key, it->nkey) == 0); cproxy_upstream_ascii_item_response(it, uc_cur, 0); psc_get_key->hits++; psc_get_key->write_bytes += it->nbytes; if (do_key_stats) { touch_key_stats(ptd, key, key_len, msec_current_time_snapshot, STATS_CMD_TYPE_REGULAR, STATS_CMD_GET_KEY, 0, 1, 0, 0, it->nbytes); } // The refcount was inc'ed by mcache_get() for us. // item_remove(it); goto loop_next; } } bool self = false; conn *c = cproxy_find_downstream_conn(d, key, key_len, &self); if (c != NULL) { if (self) { // Optimization for talking with ourselves, // to avoid extra network hop. // ptd->stats.stats.tot_optimize_self++; item *it = item_get(key, key_len); if (it != NULL) { cproxy_upstream_ascii_item_response(it, uc_cur, cas_emit); psc_get_key->hits++; psc_get_key->write_bytes += it->nbytes; if (do_key_stats) { touch_key_stats(ptd, key, key_len, msec_current_time_snapshot, STATS_CMD_TYPE_REGULAR, STATS_CMD_GET_KEY, 0, 1, 0, 0, it->nbytes); } // The refcount was inc'ed by item_get() for us. // item_remove(it); if (settings.verbose > 1) { fprintf(stderr, "optimize self multiget hit: %s\n", key); } } else { psc_get_key->misses++; if (do_key_stats) { touch_key_stats(ptd, key, key_len, msec_current_time_snapshot, STATS_CMD_TYPE_REGULAR, STATS_CMD_GET_KEY, 0, 0, 1, 0, 0); } if (settings.verbose > 1) { fprintf(stderr, "optimize self multiget miss: %s\n", key); } } goto loop_next; } // See if we've already requested this key via // the multiget hash table, in order to // de-deplicate repeated keys. // bool first_request = true; if (d->multiget != NULL) { // TODO: Use Trond's allocator here. // multiget_entry *entry = calloc(1, sizeof(multiget_entry)); if (entry != NULL) { entry->upstream_conn = uc_cur; entry->opaque = 0; entry->hits = 0; entry->next = genhash_find(d->multiget, key); genhash_update(d->multiget, key, entry); if (entry->next != NULL) { first_request = false; } } else { // TODO: Handle out of multiget entry memory. } } if (first_request) { assert(c->item == NULL); assert(c->state == conn_pause); assert(IS_PROXY(c->protocol)); assert(c->ilist != NULL); assert(c->isize > 0); if (c->msgused <= 1 && c->msgbytes <= 0) { emit_start(c, command, cmd_len); } // Provide the preceding space as optimization // for ascii-to-ascii configuration. // emit_skey(c, key - 1, key_len + 1); } else { ptd->stats.stats.tot_multiget_keys_dedupe++; if (settings.verbose > 1) { char buf[KEY_MAX_LENGTH + 10]; memcpy(buf, key, key_len); buf[key_len] = '\0'; fprintf(stderr, "%d cproxy multiget dedpue: %s\n", uc_cur->sfd, buf); } } } else { // TODO: Handle when downstream conn is down. } } loop_next: space = next_space; } uc_num++; uc_cur = uc_cur->next; } for (int i = 0; i < nconns; i++) { conn *c = d->downstream_conns[i]; if (c != NULL && (c->msgused > 1 || c->msgbytes > 0)) { emit_end(c); conn_set_state(c, conn_mwrite); c->write_and_go = conn_new_cmd; if (update_event(c, EV_WRITE | EV_PERSIST)) { nwrite++; if (uc->noreply) { c->write_and_go = conn_pause; } } else { if (settings.verbose > 1) { fprintf(stderr, "Couldn't update cproxy write event\n"); } d->ptd->stats.stats.err_oom++; cproxy_close_conn(c); } } } if (settings.verbose > 1) { fprintf(stderr, "forward multiget nwrite %d out of %d\n", nwrite, nconns); } d->downstream_used_start = nwrite; d->downstream_used = nwrite; if (cproxy_dettach_if_noreply(d, uc) == false) { d->upstream_suffix = "END\r\n"; cproxy_start_downstream_timeout(d, NULL); } return nwrite > 0; }
// process a memcached get(s) command. (we don't support CAS). void process_get_command(conn *c, token_t *tokens, size_t ntokens, bool return_cas) { char *key; size_t nkey; int i = 0; item *it; token_t *key_token = &tokens[KEY_TOKEN]; char *suffix; assert(c != NULL); if (config.alloc && c->mem_blob == NULL) { long size = config.alloc_mean + gsl_ran_gaussian(c->thread->r, config.alloc_stddev); size = size <= 0 ? 10 : size; if (config.verbose > 0) { fprintf(stderr, "allocated blob: %ld\n", size); } c->mem_blob = malloc(sizeof(char) * size); c->mem_free_delay = 0; if (config.rtt_delay) { double r = config.rtt_mean + gsl_ran_gaussian(c->thread->r, config.rtt_stddev); if (r >= config.rtt_cutoff) { int wait = r / 100; if (config.verbose > 0) { fprintf(stderr, "delay: %d\n", wait); } c->mem_free_delay = wait; conn_set_state(c, conn_mwrite); } } } // process the whole command line, (only part of it may be tokenized right now) do { // process all tokenized keys at this stage. while(key_token->length != 0) { key = key_token->value; nkey = key_token->length; if(nkey > KEY_MAX_LENGTH) { error_response(c, "CLIENT_ERROR bad command line format"); return; } // lookup key-value. it = item_get(key, nkey); // hit. if (it) { if (i >= c->isize && !conn_expand_items(c)) { item_remove(it); break; } // Construct the response. Each hit adds three elements to the // outgoing data list: // "VALUE <key> <flags> <data_length>\r\n" // "<data>\r\n" // The <data> element is stored on the connection item list, not on // the iov list. if (!conn_add_iov(c, "VALUE ", 6) != 0 || !conn_add_iov(c, ITEM_key(it), it->nkey) != 0 || !conn_add_iov(c, ITEM_suffix(it), it->nsuffix + it->nbytes) != 0) { item_remove(it); break; } if (config.verbose > 1) { fprintf(stderr, ">%d sending key %s\n", c->sfd, key); } // add item to remembered list (i.e., we've taken ownership of them // through refcounting and later must release them once we've // written out the iov associated with them). item_update(it); *(c->ilist + i) = it; i++; } key_token++; } /* * If the command string hasn't been fully processed, get the next set * of tokens. */ if(key_token->value != NULL) { ntokens = tokenize_command(key_token->value, tokens, MAX_TOKENS); key_token = tokens; } } while(key_token->value != NULL); c->icurr = c->ilist; c->ileft = i; if (config.verbose > 1) { fprintf(stderr, ">%d END\n", c->sfd); } // If the loop was terminated because of out-of-memory, it is not reliable // to add END\r\n to the buffer, because it might not end in \r\n. So we // send SERVER_ERROR instead. if (key_token->value != NULL || !conn_add_iov(c, "END\r\n", 5) != 0) { error_response(c, "SERVER_ERROR out of memory writing get response"); } else { conn_set_state(c, conn_mwrite); } }
TEST(List, Remove) { int size; Item *head = 0; //Item *item = 0; int num = 100; Item items[num]; // Initialise each item for (int i = 0; i < num; i++) { init_item(& items[i], i); } // Append to stack for (int i = 0; i < num; i++) { list_append((pList*) & head, (pList) & items[i], pnext_item, 0); size = item_size(& head); EXPECT_EQ(i+1, size); } size = item_size(& head); EXPECT_EQ(num, size); // Remove last item bool found; int i = num - 1; Item *item = & items[i]; found = item_remove(& head, item, 0); EXPECT_EQ(true, found); item_validate(item, i); // check it has gone size = item_size(& head); EXPECT_EQ(num-1, size); // can't remove it twice found = item_remove(& head, item, 0); EXPECT_EQ(false, found); // Remove the first item i = 0; item = & items[i]; found = item_remove(& head, item, 0); EXPECT_EQ(true, found); // check it has gone size = item_size(& head); EXPECT_EQ(num-2, size); // can't remove it twice found = item_remove(& head, item, 0); EXPECT_EQ(false, found); // Remove from the middle i = num / 2; item = & items[i]; found = item_remove(& head, item, 0); EXPECT_EQ(true, found); // check it has gone size = item_size(& head); EXPECT_EQ(num-3, size); // can't remove it twice found = item_remove(& head, item, 0); EXPECT_EQ(false, found); // pop the rest off while (item_pop(& head)) { } // Try removing from an empty list EXPECT_EQ(0, head); found = item_remove(& head, item, 0); EXPECT_EQ(false, found); }
void protocol_stats_foreach_write(const void *key, const void *value, void *user_data) { char *line = (char *) value; conn *uc = (conn *) user_data; int nline; cb_assert(line != NULL); cb_assert(uc != NULL); (void)key; nline = strlen(line); if (nline > 0) { item *it; if (settings.verbose > 2) { moxi_log_write("%d: cproxy_stats writing: %s\n", uc->sfd, line); } if (IS_BINARY(uc->protocol)) { token_t line_tokens[MAX_TOKENS]; size_t line_ntokens = scan_tokens(line, line_tokens, MAX_TOKENS, NULL); if (line_ntokens == 4) { uint16_t key_len = line_tokens[NAME_TOKEN].length; uint32_t data_len = line_tokens[VALUE_TOKEN].length; it = item_alloc("s", 1, 0, 0, sizeof(protocol_binary_response_stats) + key_len + data_len); if (it != NULL) { protocol_binary_response_stats *header = (protocol_binary_response_stats *) ITEM_data(it); memset(ITEM_data(it), 0, it->nbytes); header->message.header.response.magic = (uint8_t) PROTOCOL_BINARY_RES; header->message.header.response.opcode = uc->binary_header.request.opcode; header->message.header.response.keylen = (uint16_t) htons(key_len); header->message.header.response.bodylen = htonl(key_len + data_len); header->message.header.response.opaque = uc->opaque; memcpy((ITEM_data(it)) + sizeof(protocol_binary_response_stats), line_tokens[NAME_TOKEN].value, key_len); memcpy((ITEM_data(it)) + sizeof(protocol_binary_response_stats) + key_len, line_tokens[VALUE_TOKEN].value, data_len); if (add_conn_item(uc, it)) { add_iov(uc, ITEM_data(it), it->nbytes); if (settings.verbose > 2) { moxi_log_write("%d: cproxy_stats writing binary", uc->sfd); cproxy_dump_header(uc->sfd, ITEM_data(it)); } return; } item_remove(it); } } return; } it = item_alloc("s", 1, 0, 0, nline + 2); if (it != NULL) { strncpy(ITEM_data(it), line, nline); strncpy(ITEM_data(it) + nline, "\r\n", 2); if (add_conn_item(uc, it)) { add_iov(uc, ITEM_data(it), nline + 2); return; } item_remove(it); } } }