/*@null@*/ static PREFIX_STATS *stats_prefix_find(const char *key, const size_t nkey) { PREFIX_STATS *pfs; uint32_t hashval; size_t length; bool bailout = true; cb_assert(key != NULL); for (length = 0; length < nkey && key[length] != '\0'; length++) { if (key[length] == settings.prefix_delimiter) { bailout = false; break; } } if (bailout) { return NULL; } hashval = hash(key, length, 0) % PREFIX_HASH_SIZE; for (pfs = prefix_stats[hashval]; NULL != pfs; pfs = pfs->next) { if (strncmp(pfs->prefix, key, length) == 0) return pfs; } pfs = calloc(sizeof(PREFIX_STATS), 1); if (NULL == pfs) { perror("Can't allocate space for stats structure: calloc"); return NULL; } pfs->prefix = malloc(length + 1); if (NULL == pfs->prefix) { perror("Can't allocate space for copy of prefix: malloc"); free(pfs); return NULL; } strncpy(pfs->prefix, key, length); pfs->prefix[length] = '\0'; /* because strncpy() sucks */ pfs->prefix_len = length; pfs->next = prefix_stats[hashval]; prefix_stats[hashval] = pfs; num_prefixes++; total_prefix_size += (int)length; return pfs; }
/* test thing into multiple compilation units possible. */ MEMCACHED_PUBLIC_API engine_test_t* get_tests(void) { engine_test_t* testsegs[NSEGS]; engine_test_t* rv = NULL; int i = 0, j = 0; size_t num_tests = 0, pos = 0; testsegs[i++] = get_tests_0(); testsegs[i++] = get_tests_1(); testsegs[i++] = get_tests_2(); testsegs[i++] = get_tests_3(); testsegs[i++] = get_tests_4(); testsegs[i++] = get_tests_5(); testsegs[i++] = get_tests_6(); testsegs[i++] = get_tests_7(); testsegs[i++] = get_tests_8(); testsegs[i++] = get_tests_9(); cb_assert(i == NSEGS); for (i = 0; i < NSEGS; ++i) { for (j = 0; testsegs[i][j].name; ++j) { ++num_tests; } } rv = calloc(num_tests+1, sizeof(engine_test_t)); cb_assert(rv); for (i = 0; i < NSEGS; ++i) { for (j = 0; testsegs[i][j].name; ++j) { rv[pos++] = testsegs[i][j]; } } qsort(rv, num_tests, sizeof(engine_test_t), test_compare); return rv; }
void genhash_iter(genhash_t* h, void (*iterfunc)(const void* key, const void* val, void *arg), void *arg) { size_t i=0; struct genhash_entry_t *p=NULL; cb_assert(h != NULL); for(i=0; i<h->size; i++) { for(p=h->buckets[i]; p!=NULL; p=p->next) { iterfunc(p->key, p->value, arg); } } }
void cproxy_reset_stats_td(proxy_stats_td *pstd) { int j; cb_assert(pstd); cproxy_reset_stats(&pstd->stats); for (j = 0; j < STATS_CMD_TYPE_last; j++) { int k; for (k = 0; k < STATS_CMD_last; k++) { cproxy_reset_stats_cmd(&pstd->stats_cmd[j][k]); } } }
void genhash_store(genhash_t *h, const void* k, size_t klen, const void* v, size_t vlen) { size_t n=0; struct genhash_entry_t *p; cb_assert(h != NULL); n=h->ops.hashfunc(k, klen) % h->size; cb_assert((int)n >= 0); cb_assert(n < h->size); p=calloc(1, sizeof(struct genhash_entry_t)); cb_assert(p); p->key=dup_key(h, k, klen); p->nkey = klen; p->value=dup_value(h, v, vlen); p->nvalue = vlen; p->next=h->buckets[n]; h->buckets[n]=p; }
void cproxy_binary_uncork_cmds(downstream *d, conn *uc) { cb_assert(d != NULL); cb_assert(uc != NULL); if (settings.verbose > 2) { moxi_log_write("%d: cproxy_binary_uncork_cmds\n", uc->sfd); } int n = 0; while (uc->corked != NULL) { bin_cmd *next = uc->corked->next; item *it = uc->corked->request_item; if (it != NULL) { b2b_forward_item(uc, d, it); n++; } if (uc->corked->request_item != NULL) { item_remove(uc->corked->request_item); } if (uc->corked->response_item != NULL) { item_remove(uc->corked->response_item); } free(uc->corked); uc->corked = next; } if (settings.verbose > 2) { moxi_log_write("%d: cproxy_binary_uncork_cmds, uncorked %d\n", uc->sfd, n); } }
/** * Depending on our configuration, we can optimize SET's * on certain keys by making them fire-and-forget and * immediately transmitting a success response to the * upstream client. */ bool cproxy_optimize_set_ascii(downstream *d, conn *uc, char *key, int key_len) { cb_assert(d); cb_assert(d->ptd); cb_assert(d->ptd->proxy); cb_assert(uc); cb_assert(uc->next == NULL); if (d->ptd->behavior_pool.base.optimize_set[0] == '\0') { return false; } if (matcher_check(&d->ptd->proxy->optimize_set_matcher, key, key_len, false)) { d->upstream_conn = NULL; d->upstream_suffix = NULL; d->upstream_suffix_len = 0; d->upstream_status = PROTOCOL_BINARY_RESPONSE_SUCCESS; d->upstream_retry = 0; d->target_host_ident = NULL; out_string(uc, "STORED"); if (!update_event(uc, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 1) { moxi_log_write("ERROR: Can't update upstream write event\n"); } d->ptd->stats.stats.err_oom++; cproxy_close_conn(uc); } return true; } return false; }
void notify_io_complete(const void *cookie, ENGINE_ERROR_CODE status) { struct conn *conn = (struct conn *)cookie; LIBEVENT_THREAD *thr; int notify; cb_assert(conn); thr = conn->thread; cb_assert(thr); settings.extensions.logger->log(EXTENSION_LOG_DEBUG, NULL, "Got notify from %d, status %x\n", conn->sfd, status); LOCK_THREAD(thr); conn->aiostat = status; notify = add_conn_to_pending_io_list(conn); UNLOCK_THREAD(thr); /* kick the thread in the butt */ if (notify) { notify_thread(thr); } }
static void incr_test_main(void *arg) { ENGINE_HANDLE *h = arg; ENGINE_HANDLE_V1 *h1 = arg; void *key = "incr_test_key"; uint64_t cas = 0; uint64_t res = 0; int ii; for (ii = 0; ii < 1000; ++ii) { cb_assert(h1->arithmetic(h, NULL, key, (int)strlen(key), false, false, 1, 0, 0, &cas, PROTOCOL_BINARY_RAW_BYTES, &res, 0 ) == ENGINE_SUCCESS); } }
int genhash_delete(genhash_t* h, const void* k) { struct genhash_entry_t *deleteme=NULL; int n=0; int rv=0; cb_assert(h != NULL); n=h->ops.hashfunc(k) % h->size; cb_assert(n >= 0); cb_assert(n < (int) h->size); if(h->buckets[n] != NULL) { /* Special case the first one */ if(h->ops.hasheq(h->buckets[n]->key, k)) { deleteme=h->buckets[n]; h->buckets[n]=deleteme->next; } else { struct genhash_entry_t *p=NULL; for(p=h->buckets[n]; deleteme==NULL && p->next != NULL; p=p->next) { if(h->ops.hasheq(p->next->key, k)) { deleteme=p->next; p->next=deleteme->next; } } } } if(deleteme != NULL) { h->ops.freeKey(deleteme->key); h->ops.freeValue(deleteme->value); free(deleteme); rv++; } return rv; }
static void *get_response(BIO *bio, protocol_binary_response_no_extras *res) { uint32_t vallen; ensure_recv(bio, res, sizeof(*res)); vallen = ntohl(res->message.header.response.bodylen); if (vallen == 0) { return NULL; } else { void *buffer = malloc(vallen); cb_assert(buffer != NULL); ensure_recv(bio, buffer, vallen); return buffer; } }
void do_item_update(struct default_engine *engine, hash_item *it) { rel_time_t current_time = engine->server.core->get_current_time(); MEMCACHED_ITEM_UPDATE(hash_key_get_client_key(item_get_key(it)), hash_key_get_client_key_len(item_get_key(it)), it->nbytes); if (it->time < current_time - ITEM_UPDATE_INTERVAL) { cb_assert((it->iflag & ITEM_SLABBED) == 0); if ((it->iflag & ITEM_LINKED) != 0) { item_unlink_q(engine, it); it->time = current_time; item_link_q(engine, it); } } }
static void do_ssl_test(BIO *bio) { uint32_t datalen = 512*1024; void *data = malloc(datalen); char *rcv = NULL; const char *key = "Hello World"; int ii; for (ii = 0; ii < 1024; ++ii) { fprintf(stdout, "%u\n", ii); store(bio, key, (uint16_t)strlen(key), data, datalen); rcv = fetch(bio, key, (uint16_t)strlen(key)); cb_assert(memcmp(data, rcv + 4, datalen) == 0); free(rcv); } }
void slabs_adjust_mem_requested(struct default_engine *engine, unsigned int id, size_t old, size_t ntotal) { slabclass_t *p; cb_mutex_enter(&engine->slabs.lock); if (id < POWER_SMALLEST || id > engine->slabs.power_largest) { EXTENSION_LOGGER_DESCRIPTOR *logger; logger = (void*)engine->server.extension->get_extension(EXTENSION_LOGGER); logger->log(EXTENSION_LOG_WARNING, NULL, "Internal error! Invalid slab class\n"); cb_assert(false); } p = &engine->slabs.slabclass[id]; p->requested = p->requested - old + ntotal; cb_mutex_exit(&engine->slabs.lock); }
static view_btree_value_t *test_view_btree_value_decoding(const char *value_bin, size_t len) { view_btree_value_t *v = NULL; cb_assert(decode_view_btree_value(value_bin, len, &v) == COUCHSTORE_SUCCESS); cb_assert(v != NULL); cb_assert(v->partition == 10); cb_assert(v->num_values == 2); cb_assert(v->values[0].size == 4); cb_assert(memcmp(v->values[0].buf, "6155", v->values[0].size) == 0); cb_assert(v->values[1].size == 4); cb_assert(memcmp(v->values[1].buf, "6154", v->values[0].size) == 0); return v; }
/* * Make sure we can successfully retrieve the item info struct for an item and * that the contents of the item_info are as expected. */ static enum test_result get_item_info_test(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) { item *test_item = NULL; char *key = "get_item_info_test_key"; uint64_t cas = 0; const rel_time_t exp = 1; item_info ii; ii.nvalue = 1; cb_assert(h1->allocate(h, NULL, &test_item, key, strlen(key), 1,0, exp, PROTOCOL_BINARY_RAW_BYTES) == ENGINE_SUCCESS); cb_assert(h1->store(h, NULL, test_item, &cas, OPERATION_SET,0) == ENGINE_SUCCESS); /* Had this been actual code, there'd be a connection here */ cb_assert(h1->get_item_info(h, NULL, test_item, &ii) == true); cb_assert(ii.cas == cas); cb_assert(ii.flags == 0); cb_assert(strcmp(key,ii.key) == 0); cb_assert(ii.nkey == strlen(key)); cb_assert(ii.nbytes == 1); cb_assert(ii.exptime == exp); h1->release(h, NULL, test_item); return SUCCESS; }
bool safe_strtol(const char *str, int32_t *out) { char *endptr; long l; cb_assert(out != NULL); errno = 0; *out = 0; l = strtol(str, &endptr, 10); if (errno == ERANGE) { return false; } if (xisspace(*endptr) || (*endptr == '\0' && endptr != str)) { *out = l; return true; } return false; }
int genhash_clear(genhash_t *h) { size_t i = 0; int rv = 0; cb_assert(h != NULL); for(i = 0; i < h->size; i++) { while(h->buckets[i]) { struct genhash_entry_t *p = NULL; p = h->buckets[i]; h->buckets[i] = p->next; free_item(h, p); } } return rv; }
uint32_t libhashkit_digest(const char *key, size_t key_length, hashkit_hash_algorithm_t hash_algorithm) { switch (hash_algorithm) { case HASHKIT_HASH_DEFAULT: return libhashkit_one_at_a_time(key, key_length); case HASHKIT_HASH_MD5: return libhashkit_md5(key, key_length); case HASHKIT_HASH_CRC: return libhashkit_crc32(key, key_length); case HASHKIT_HASH_FNV1_64: return libhashkit_fnv1_64(key, key_length); case HASHKIT_HASH_FNV1A_64: return libhashkit_fnv1a_64(key, key_length); case HASHKIT_HASH_FNV1_32: return libhashkit_fnv1_32(key, key_length); case HASHKIT_HASH_FNV1A_32: return libhashkit_fnv1a_32(key, key_length); case HASHKIT_HASH_HSIEH: #ifdef HAVE_HSIEH_HASH return libhashkit_hsieh(key, key_length); #else return 1; #endif case HASHKIT_HASH_MURMUR: #ifdef HAVE_MURMUR_HASH return libhashkit_murmur(key, key_length); #else return 1; #endif case HASHKIT_HASH_JENKINS: return libhashkit_jenkins(key, key_length); case HASHKIT_HASH_CUSTOM: case HASHKIT_HASH_MAX: default: #ifdef HAVE_DEBUG fprintf(stderr, "hashkit_hash_t was extended but libhashkit_generate_value was not updated\n"); fflush(stderr); cb_assert(0); #endif break; } return 1; }
void assoc_delete(struct default_engine *engine, uint32_t hash, const char *key, const size_t nkey) { hash_item **before = _hashitem_before(engine, hash, key, nkey); if (*before) { hash_item *nxt; engine->assoc.hash_items--; /* The DTrace probe cannot be triggered as the last instruction * due to possible tail-optimization by the compiler */ MEMCACHED_ASSOC_DELETE(key, nkey, engine->assoc.hash_items); nxt = (*before)->h_next; (*before)->h_next = 0; /* probably pointless, but whatever. */ *before = nxt; return; } /* Note: we never actually get here. the callers don't delete things they can't find. */ cb_assert(*before != 0); }
int genhash_clear(genhash_t *h) { size_t i = 0; cb_assert(h != NULL); for(i = 0; i < h->size; i++) { while(h->buckets[i]) { struct genhash_entry_t *p = NULL; p = h->buckets[i]; h->buckets[i] = p->next; h->ops.freeKey(p->key); h->ops.freeValue(p->value); free(p); } } return 0; }
/*@null@*/ char *stats_prefix_dump(int *length) { const char *format = "PREFIX %s get %llu hit %llu set %llu del %llu\r\n"; PREFIX_STATS *pfs; char *buf; int i, pos; size_t size = 0, written = 0, total_written = 0; /* * Figure out how big the buffer needs to be. This is the sum of the * lengths of the prefixes themselves, plus the size of one copy of * the per-prefix output with 20-digit values for all the counts, * plus space for the "END" at the end. */ STATS_LOCK(); size = strlen(format) + total_prefix_size + num_prefixes * (strlen(format) - 2 /* %s */ + 4 * (20 - 4)) /* %llu replaced by 20-digit num */ + sizeof("END\r\n"); buf = malloc(size); if (NULL == buf) { perror("Can't allocate stats response: malloc"); STATS_UNLOCK(); return NULL; } pos = 0; for (i = 0; i < PREFIX_HASH_SIZE; i++) { for (pfs = prefix_stats[i]; NULL != pfs; pfs = pfs->next) { written = snprintf(buf + pos, size-pos, format, pfs->prefix, pfs->num_gets, pfs->num_hits, pfs->num_sets, pfs->num_deletes); pos += (int)written; total_written += written; cb_assert(total_written < size); } } STATS_UNLOCK(); memcpy(buf + pos, "END\r\n", 6); *length = pos + 5; return buf; }
bool b2b_forward_item_vbucket(conn *uc, downstream *d, item *it, conn *c, int vbucket) { protocol_binary_request_header *req; cb_assert(d != NULL); cb_assert(d->ptd != NULL); cb_assert(uc != NULL); cb_assert(uc->next == NULL); cb_assert(uc->noreply == false); cb_assert(c != NULL); /* Assuming we're already connected to downstream. */ if (settings.verbose > 2) { moxi_log_write("%d: b2b_forward_item_vbucket %x to %d, vbucket %d\n", uc->sfd, uc->cmd, c->sfd, vbucket); } req = (protocol_binary_request_header *) ITEM_data(it); if (vbucket >= 0) { req->request.reserved = htons(vbucket); } if (add_conn_item(c, it) == true) { /* The caller keeps its refcount, and we need our own. */ it->refcount++; if (add_iov(c, ITEM_data(it), it->nbytes) == 0) { conn_set_state(c, conn_mwrite); c->write_and_go = conn_new_cmd; if (update_event(c, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 2) { moxi_log_write("%d: b2b_forward %x to %d success\n", uc->sfd, uc->cmd, c->sfd); } return true; } } } d->ptd->stats.stats.err_oom++; cproxy_close_conn(c); return false; }
static const char* createStringFromJSON(const char** in, size_t *length, bool *freeWhenDone) { char* buf; char* dst; char c; /* Scan the JSON string to find its length and whether it contains escapes: */ const char* start = ++*in; unsigned escapes = 0; const char* str; for (str = start; *str != '"'; ++str) { if (*str == '\\') { ++str; if (*str == 'u') { escapes += 5; /* \uxxxx adds 5 bytes */ str += 4; } else escapes += 1; } } *in = str + 1; *length = str - start; *freeWhenDone = false; if (escapes > 0) { *length -= escapes; buf = malloc(*length); dst = buf; for (str = start; (c = *str) != '"'; ++str) { if (c == '\\') c = ConvertJSONEscape(&str); *dst++ = c; } cb_assert(dst - buf == (int)*length); start = buf; *freeWhenDone = true; } return start; }
/* Note: this isn't an assoc_update. The key must not already exist to call this */ int assoc_insert(struct default_engine *engine, uint32_t hash, hash_item *it) { unsigned int oldbucket; cb_assert(assoc_find(engine, hash, item_get_key(it), it->nkey) == 0); /* shouldn't have duplicately named things defined */ if (engine->assoc.expanding && (oldbucket = (hash & hashmask(engine->assoc.hashpower - 1))) >= engine->assoc.expand_bucket) { it->h_next = engine->assoc.old_hashtable[oldbucket]; engine->assoc.old_hashtable[oldbucket] = it; } else { it->h_next = engine->assoc.primary_hashtable[hash & hashmask(engine->assoc.hashpower)]; engine->assoc.primary_hashtable[hash & hashmask(engine->assoc.hashpower)] = it; } engine->assoc.hash_items++; if (! engine->assoc.expanding && engine->assoc.hash_items > (hashsize(engine->assoc.hashpower) * 3) / 2) { assoc_expand(engine); } MEMCACHED_ASSOC_INSERT(item_get_key(it), it->nkey, engine->assoc.hash_items); return 1; }
static double readNumber(const char* start, const char* end, char** endOfNumber) { /* First copy the string into a zero-terminated buffer so we can safely call strtod: */ char buf[50]; char* endInStr; double result; size_t len; char* str; cb_assert(end > start); len = end - start; str = (len < sizeof(buf)) ? buf : malloc(len + 1); if (!str) return 0.0; memcpy(str, start, len); str[len] = '\0'; result = strtod(str, &endInStr); *endOfNumber = (char*)start + (endInStr - str); if (len >= sizeof(buf)) free(str); return result; }
int do_item_link(struct default_engine *engine, hash_item *it) { const hash_key* key = item_get_key(it); MEMCACHED_ITEM_LINK(hash_key_get_client_key(key), hash_key_get_client_key_len(key), it->nbytes); cb_assert((it->iflag & (ITEM_LINKED|ITEM_SLABBED)) == 0); it->iflag |= ITEM_LINKED; it->time = engine->server.core->get_current_time(); assoc_insert(engine, crc32c(hash_key_get_key(key), hash_key_get_key_len(key), 0), it); cb_mutex_enter(&engine->stats.lock); engine->stats.curr_bytes += ITEM_ntotal(engine, it); engine->stats.curr_items += 1; engine->stats.total_items += 1; cb_mutex_exit(&engine->stats.lock); /* Allocate a new CAS ID on link. */ item_set_cas(NULL, NULL, it, get_cas_id()); item_link_q(engine, it); return 1; }
static void storeItem(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, ENGINE_STORE_OPERATION op) { item *it = NULL; uint64_t cas = 0; char *value = "0"; const int flags = 0; const void *cookie = NULL; size_t vlen; ENGINE_ERROR_CODE rv; item_info info; if (op == OPERATION_APPEND) { value = "-suffix"; } else if (op == OPERATION_PREPEND) { value = "prefix-"; } vlen = strlen(value); rv = h1->allocate(h, cookie, &it, key, strlen(key), vlen, flags, expiry, PROTOCOL_BINARY_RAW_BYTES); cb_assert(rv == ENGINE_SUCCESS); info.nvalue = 1; if (!h1->get_item_info(h, cookie, it, &info)) { abort(); } memcpy(info.value[0].iov_base, value, vlen); h1->item_set_cas(h, cookie, it, 0); rv = h1->store(h, cookie, it, &cas, op, 0); hasError = rv != ENGINE_SUCCESS; }
genhash_t* genhash_init(int est, struct hash_ops ops) { genhash_t* rv=NULL; int size=0; if (est < 1) { return NULL; } cb_assert(ops.hashfunc != NULL); cb_assert(ops.hasheq != NULL); cb_assert(ops.dupKey != NULL); cb_assert(ops.dupValue != NULL); cb_assert(ops.freeKey != NULL); cb_assert(ops.freeValue != NULL); size=estimate_table_size(est); rv=calloc(1, sizeof(genhash_t) + (size * sizeof(struct genhash_entry_t *))); cb_assert(rv != NULL); rv->size=size; rv->ops=ops; return rv; }
static void cproxy_sasl_plain_auth(conn *c, char *req_bytes) { protocol_binary_request_header *req; char *key; int keylen; int bodylen; char *clientin; unsigned int clientinlen; proxy_td *ptd = c->extra; cb_assert(ptd != NULL); cb_assert(ptd->proxy != NULL); cb_assert(ptd->proxy->main != NULL); /* Authenticate an upstream connection. */ req = (protocol_binary_request_header *) req_bytes; key = ((char *) req) + sizeof(*req) + req->request.extlen; keylen = ntohs(req->request.keylen); bodylen = ntohl(req->request.bodylen); /* The key is the sasl mech. */ if (keylen != 5 || memcmp(key, "PLAIN", 5) != 0) { /* 5 == strlen("PLAIN"). */ write_bin_error(c, PROTOCOL_BINARY_RESPONSE_AUTH_ERROR, 0); return; } clientin = key + keylen; clientinlen = bodylen - keylen - req->request.extlen; /* The clientin string looks like "[authzid]\0username\0password". */ while (clientinlen > 0 && clientin[0] != '\0') { /* Skip authzid. */ clientin++; clientinlen--; } if (clientinlen > 2 && clientinlen < 128 && clientin[0] == '\0') { const char *username = clientin + 1; char password[256]; int uslen = strlen(username); int pwlen = clientinlen - 2 - uslen; if (pwlen < (int) sizeof(password)) { proxy *p; memcpy(password, clientin + 2 + uslen, pwlen); password[pwlen] = '\0'; p = cproxy_find_proxy_by_auth(ptd->proxy->main, username, password); if (p != NULL) { proxy_td *ptd_target = cproxy_find_thread_data(p, cb_thread_self()); if (ptd_target != NULL) { c->extra = ptd_target; write_bin_response(c, "Authenticated", 0, 0, strlen("Authenticated")); if (settings.verbose > 2) { moxi_log_write("<%d sasl authenticated for %s\n", c->sfd, username); } return; } else { if (settings.verbose > 2) { moxi_log_write("<%d sasl auth failed on ptd for %s\n", c->sfd, username); } } } else { if (settings.verbose > 2) { moxi_log_write("<%d sasl auth failed for %s (%d)\n", c->sfd, username, pwlen); } } } else { if (settings.verbose > 2) { moxi_log_write("<%d sasl auth failed for %s with empty password\n", c->sfd, username); } } } else { if (settings.verbose > 2) { moxi_log_write("<%d sasl auth failed with malformed PLAIN data\n", c->sfd); } } /* TODO: If authentication failed, we should consider */ /* reassigning the connection to the NULL_BUCKET. */ write_bin_error(c, PROTOCOL_BINARY_RESPONSE_AUTH_ERROR, 0); }