static int cdb_readv(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) { if (!db || !key || !val) { return kr_error(EINVAL); } struct memcached_cli *cli = db; /* Convert to libmemcached query format */ assert(maxcount < 1000); /* Sane upper bound */ const char *keys [maxcount]; size_t lengths [maxcount]; for (int i = 0; i < maxcount; ++i) { keys[i] = key[i].data; lengths[i] = key[i].len; } /* Execute multiple get and retrieve results */ memcached_return_t status = memcached_mget(cli->handle, keys, lengths, maxcount); memcached_result_free(&cli->res); memcached_result_create(cli->handle, &cli->res); for (int i = 0; i < maxcount; ++i) { memcached_result_st *res = memcached_fetch_result(cli->handle, &cli->res, &status); if (!res) { /* Less results than expected */ return kr_error(ENOENT); } val[i].len = memcached_result_length(res); val[i].data = (void *)memcached_result_value(res); } return 0; }
static int cdb_init(knot_db_t **db, struct kr_cdb_opts *opts, knot_mm_t *pool) { if (!db || !opts) { return kr_error(EINVAL); } struct memcached_cli *cli = malloc(sizeof(*cli)); if (!cli) { return kr_error(ENOMEM); } memset(cli, 0, sizeof(*cli)); /* Make sure we're running on binary protocol, as the * textual protocol is broken for binary keys. */ auto_free char *config_str = kr_strcatdup(2, opts->path, " --BINARY-PROTOCOL"); cli->handle = memcached(config_str, strlen(config_str)); if (!cli->handle) { free(cli); return kr_error(EIO); } /* Create result set */ memcached_result_st *res = memcached_result_create(cli->handle, &cli->res); if (!res) { memcached_free(cli->handle); free(cli); return kr_error(ENOMEM); } *db = cli; return 0; }
memcached_st *memcached_create(memcached_st *ptr) { memcached_result_st *result_ptr; #ifdef _WIN32 WSADATA wsaData; WSAStartup(MAKEWORD(2, 0), &wsaData); #endif if (ptr == NULL) { ptr= (memcached_st *)malloc(sizeof(memcached_st)); if (!ptr) return NULL; /* MEMCACHED_MEMORY_ALLOCATION_FAILURE */ memset(ptr, 0, sizeof(memcached_st)); ptr->is_allocated= true; } else { memset(ptr, 0, sizeof(memcached_st)); } result_ptr= memcached_result_create(ptr, &ptr->result); WATCHPOINT_ASSERT(result_ptr); ptr->poll_timeout= MEMCACHED_DEFAULT_TIMEOUT; ptr->connect_timeout= MEMCACHED_DEFAULT_TIMEOUT; ptr->retry_timeout= 0; ptr->distribution= MEMCACHED_DISTRIBUTION_MODULA; /* TODO, Document why we picked these defaults */ ptr->io_msg_watermark= 500; ptr->io_bytes_watermark= 65 * 1024; return ptr; }
memcached_result_st *memcached_fetch_result(memcached_st *ptr, memcached_result_st *result, memcached_return *error) { if (result == NULL) result= memcached_result_create(ptr, NULL); #ifdef UNUSED if (ptr->flags & MEM_NO_BLOCK) memcached_io_preread(ptr); #endif while (ptr->cursor_server < ptr->number_of_hosts) { char buffer[MEMCACHED_DEFAULT_COMMAND_SIZE]; if (memcached_server_response_count(&ptr->hosts[ptr->cursor_server]) == 0) { ptr->cursor_server++; continue; } *error= memcached_response(&ptr->hosts[ptr->cursor_server], buffer, MEMCACHED_DEFAULT_COMMAND_SIZE, result); if (*error == MEMCACHED_END) /* END means that we move on to the next */ { memcached_server_response_reset(&ptr->hosts[ptr->cursor_server]); ptr->cursor_server++; continue; } else if (*error == MEMCACHED_SUCCESS) return result; else return NULL; } /* We have completed reading data */ if (result->is_allocated) memcached_result_free(result); else memcached_string_reset(&result->value); ptr->cursor_server= 0; return NULL; }
void test_memcached_cas(void) { memcached_return rc; const char *key = "caskey"; size_t key_len = strlen(key); const char* keys[2] = { (char *)key, NULL }; size_t key_lens[2] = { key_len, 0 }; memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_SUPPORT_CAS, 1); grn_test_memcached_assert( memcached_set(memc, key, key_len, "cas test", 8, 0, 0xdeadbeefU), cut_message("memcached set failed.")); { uint64_t cas; memcached_result_st *results; memcached_result_st results_obj; results = memcached_result_create(memc, &results_obj); grn_test_memcached_assert( memcached_mget(memc, keys, key_lens, 1), cut_message("memcached mget failed.")); results = memcached_fetch_result(memc, &results_obj, &rc); cut_assert_not_null(results, cut_message("memcached fetch result failed.")); cas = memcached_result_cas(results); cut_assert_operator(cas, !=, 0, cut_message("memcached cas value is non-zero.")); grn_test_memcached_assert( memcached_cas(memc, key, key_len, "cas changed", 12, 0, 0, cas), cut_message("memcached cas failed.")); grn_test_memcached_assert_equal_rc( MEMCACHED_NOTSTORED, memcached_cas(memc, key, key_len, "cas changed", 12, 0, 0, cas), cut_message("memcached cas value is same.")); memcached_result_free(&results_obj); } }
void test_memcached_cas(void) { memcached_return rc; const char *key = "caskey"; size_t key_len = strlen(key); char* keys[2] = { (char *)key, NULL }; size_t key_lens[2] = { key_len, 0 }; memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_SUPPORT_CAS, 1); rc = memcached_set(memc, key, key_len, "cas test", 8, 0, 0xdeadbeefU); cut_set_message("memcached set failed."); cut_assert_equal_int(MEMCACHED_SUCCESS, rc); { uint64_t cas; memcached_result_st *results; memcached_result_st results_obj; results = memcached_result_create(memc, &results_obj); rc = memcached_mget(memc, keys, key_lens, 1); cut_set_message("memcached mget failed."); cut_assert_equal_int(MEMCACHED_SUCCESS, rc); results = memcached_fetch_result(memc, &results_obj, &rc); cas = memcached_result_cas(results); cut_set_message("memcached cas value is non-zero."); cut_assert_operator_int(cas, !=, 0); rc = memcached_cas(memc, key, key_len, "cas changed", 12, 0, 0, cas); cut_set_message("memcached cas failed."); cut_assert_equal_int(MEMCACHED_SUCCESS, rc); rc = memcached_cas(memc, key, key_len, "cas changed", 12, 0, 0, cas); cut_set_message("memcached cas value is same."); /* TODO: fix rc after libmemcached fix */ cut_assert_equal_int(MEMCACHED_PROTOCOL_ERROR, rc); /* cut_assert_equal_int(MEMCACHED_DATA_EXISTS, rc); */ memcached_result_free(&results_obj); } }
/// Retrieve the AoR data for a given SIP URI, creating it if there isn't /// any already, and returning NULL if we can't get a connection. AoR* MemcachedStore::get_aor_data(const std::string& aor_id) ///< the SIP URI { memcached_return_t rc; MemcachedAoR* aor_data = NULL; // Try to get a connection struct timespec wait_time; wait_time.tv_sec = 0; wait_time.tv_nsec = 100 * 1000 * 1000; memcached_st* st = memcached_pool_fetch(_pool, &wait_time, &rc); if (st != NULL) { // Got one: use it. const char* key_ptr = aor_id.data(); const size_t key_len = aor_id.length(); rc = memcached_mget(st, &key_ptr, &key_len, 1); if (memcached_success(rc)) { memcached_result_st result; memcached_result_create(st, &result); memcached_fetch_result(st, &result, &rc); if (memcached_success(rc)) { aor_data = deserialize_aor(std::string(memcached_result_value(&result), memcached_result_length(&result))); aor_data->set_cas(memcached_result_cas(&result)); int now = time(NULL); expire_bindings(aor_data, now); } else { // AoR does not exist, so create it. aor_data = new MemcachedAoR(); } } memcached_pool_release(_pool, st); } return (AoR*)aor_data; }
memcached_st *memcached_create(memcached_st *ptr) { if (ptr == NULL) { ptr= (memcached_st *)malloc(sizeof(memcached_st)); if (! ptr) { return NULL; /* MEMCACHED_MEMORY_ALLOCATION_FAILURE */ } ptr->options.is_allocated= true; } else { ptr->options.is_allocated= false; } #if 0 memcached_set_purging(ptr, false); memcached_set_processing_input(ptr, false); #endif if (! _memcached_init(ptr)) { memcached_free(ptr); return NULL; } if (! memcached_result_create(ptr, &ptr->result)) { memcached_free(ptr); return NULL; } WATCHPOINT_ASSERT_INITIALIZED(&ptr->result); return ptr; }
memcached_return_t BaseMemcachedStore::get_from_replica(memcached_st* replica, const char* key_ptr, const size_t key_len, std::string& data, uint64_t& cas) { memcached_return_t rc = MEMCACHED_ERROR; cas = 0; // We must use memcached_mget because memcached_get does not retrieve CAS // values. rc = memcached_mget(replica, &key_ptr, &key_len, 1); if (memcached_success(rc)) { // memcached_mget command was successful, so retrieve the result. TRC_DEBUG("Fetch result"); memcached_result_st result; memcached_result_create(replica, &result); memcached_fetch_result(replica, &result, &rc); if (memcached_success(rc)) { // Found a record, so exit the read loop. TRC_DEBUG("Found record on replica"); // Copy the record into a string. std::string::assign copies its // arguments when used with a char*, so we can free the result // afterwards. data.assign(memcached_result_value(&result), memcached_result_length(&result)); cas = memcached_result_cas(&result); } memcached_result_free(&result); } return rc; }
Variant c_Memcache::t_get(CVarRef key, VRefParam flags /*= null*/) { if (key.is(KindOfArray)) { std::vector<const char *> real_keys; std::vector<size_t> key_len; Array keyArr = key.toArray(); real_keys.reserve(keyArr.size()); key_len.reserve(keyArr.size()); for (ArrayIter iter(keyArr); iter; ++iter) { real_keys.push_back(const_cast<char *>(iter.second().toString().c_str())); key_len.push_back(iter.second().toString().length()); } if (!real_keys.empty()) { const char *payload = NULL; size_t payload_len = 0; uint32_t flags = 0; const char *res_key = NULL; size_t res_key_len = 0; memcached_result_st result; memcached_return_t ret = memcached_mget(&m_memcache, &real_keys[0], &key_len[0], real_keys.size()); memcached_result_create(&m_memcache, &result); Array return_val; while ((memcached_fetch_result(&m_memcache, &result, &ret)) != NULL) { if (ret != MEMCACHED_SUCCESS) { // should probably notify about errors continue; } payload = memcached_result_value(&result); payload_len = memcached_result_length(&result); flags = memcached_result_flags(&result); res_key = memcached_result_key_value(&result); res_key_len = memcached_result_key_length(&result); return_val.set(String(res_key, res_key_len, CopyString), memcache_fetch_from_storage(payload, payload_len, flags)); } memcached_result_free(&result); return return_val; } } else { char *payload = NULL; size_t payload_len = 0; uint32_t flags = 0; memcached_return_t ret; String skey = key.toString(); if (skey.length() == 0) { return false; } payload = memcached_get(&m_memcache, skey.c_str(), skey.length(), &payload_len, &flags, &ret); /* This is for historical reasons from libmemcached*/ if (ret == MEMCACHED_END) { ret = MEMCACHED_NOTFOUND; } if (ret == MEMCACHED_NOTFOUND) { return false; } Variant retval = memcache_fetch_from_storage(payload, payload_len, flags); free(payload); return retval; } return false; }
static Variant HHVM_METHOD(Memcache, get, const Variant& key, VRefParam flags /*= null*/) { auto data = Native::data<MemcacheData>(this_); if (key.is(KindOfArray)) { std::vector<const char *> real_keys; std::vector<size_t> key_len; Array keyArr = key.toArray(); real_keys.reserve(keyArr.size()); key_len.reserve(keyArr.size()); for (ArrayIter iter(keyArr); iter; ++iter) { auto key = iter.second().toString(); String serializedKey = memcache_prepare_key(key); char *k = new char[serializedKey.length()+1]; std::strcpy(k, serializedKey.c_str()); real_keys.push_back(k); key_len.push_back(serializedKey.length()); } if (!real_keys.empty()) { const char *payload = NULL; size_t payload_len = 0; uint32_t flags = 0; const char *res_key = NULL; size_t res_key_len = 0; memcached_result_st result; memcached_return_t ret = memcached_mget(&data->m_memcache, &real_keys[0], &key_len[0], real_keys.size()); memcached_result_create(&data->m_memcache, &result); Array return_val; while ((memcached_fetch_result(&data->m_memcache, &result, &ret)) != nullptr) { if (ret != MEMCACHED_SUCCESS) { // should probably notify about errors continue; } payload = memcached_result_value(&result); payload_len = memcached_result_length(&result); flags = memcached_result_flags(&result); res_key = memcached_result_key_value(&result); res_key_len = memcached_result_key_length(&result); return_val.set(String(res_key, res_key_len, CopyString), memcache_fetch_from_storage(payload, payload_len, flags)); } memcached_result_free(&result); for ( size_t i = 0 ; i < real_keys.size() ; i++ ) { delete [] real_keys[i]; } return return_val; } } else { char *payload = NULL; size_t payload_len = 0; uint32_t flags = 0; memcached_return_t ret; String serializedKey = memcache_prepare_key(key.toString()); if (serializedKey.length() == 0) { return false; } payload = memcached_get(&data->m_memcache, serializedKey.c_str(), serializedKey.length(), &payload_len, &flags, &ret); /* This is for historical reasons from libmemcached*/ if (ret == MEMCACHED_END) { ret = MEMCACHED_NOTFOUND; } if (ret == MEMCACHED_NOTFOUND) { return false; } Variant retval = memcache_fetch_from_storage(payload, payload_len, flags); free(payload); return retval; } return false; }
/// Retrieve the data for a given namespace and key. Store::Status MemcachedStore::get_data(const std::string& table, const std::string& key, std::string& data, uint64_t& cas, SAS::TrailId trail) { Store::Status status = Store::Status::OK; // Construct the fully qualified key. std::string fqkey = table + "\\\\" + key; const char* key_ptr = fqkey.data(); const size_t key_len = fqkey.length(); const std::vector<memcached_st*>& replicas = get_replicas(fqkey, Op::READ); if (trail != 0) { SAS::Event start(trail, SASEvent::MEMCACHED_GET_START, 0); start.add_var_param(fqkey); SAS::report_event(start); } LOG_DEBUG("%d read replicas for key %s", replicas.size(), fqkey.c_str()); // Read from all replicas until we get a positive result. memcached_return_t rc = MEMCACHED_ERROR; bool active_not_found = false; size_t failed_replicas = 0; size_t ii; // If we only have one replica, we should try it twice - // libmemcached won't notice a dropped TCP connection until it tries // to make a request on it, and will fail the request then // reconnect, so the second attempt could still work. size_t attempts = (replicas.size() == 1) ? 2 : replicas.size(); for (ii = 0; ii < attempts; ++ii) { size_t replica_idx; if ((replicas.size() == 1) && (ii == 1)) { if (rc != MEMCACHED_CONNECTION_FAILURE) { // This is a legitimate error, not a server failure, so we // shouldn't retry. break; } replica_idx = 0; LOG_WARNING("Failed to read from sole memcached replica: retrying once"); } else { replica_idx = ii; } // We must use memcached_mget because memcached_get does not retrieve CAS // values. LOG_DEBUG("Attempt to read from replica %d (connection %p)", replica_idx, replicas[replica_idx]); rc = memcached_mget(replicas[replica_idx], &key_ptr, &key_len, 1); if (memcached_success(rc)) { // memcached_mget command was successful, so retrieve the result. LOG_DEBUG("Fetch result"); memcached_result_st result; memcached_result_create(replicas[replica_idx], &result); memcached_fetch_result(replicas[replica_idx], &result, &rc); if (memcached_success(rc)) { // Found a record, so exit the read loop. LOG_DEBUG("Found record on replica %d", replica_idx); data.assign(memcached_result_value(&result), memcached_result_length(&result)); cas = (active_not_found) ? 0 : memcached_result_cas(&result); // std::string::assign copies its arguments when used with a // char*, so this is safe. memcached_result_free(&result); break; } else { // Free the result and continue the read loop. memcached_result_free(&result); } } if (rc == MEMCACHED_NOTFOUND) { // Failed to find a record on an active replica. Flag this so if we do // find data on a later replica we can reset the cas value returned to // zero to ensure a subsequent write will succeed. LOG_DEBUG("Read for %s on replica %d returned NOTFOUND", fqkey.c_str(), replica_idx); active_not_found = true; } else { // Error from this node, so consider it inactive. LOG_DEBUG("Read for %s on replica %d returned error %d (%s)", fqkey.c_str(), replica_idx, rc, memcached_strerror(replicas[replica_idx], rc)); ++failed_replicas; } } if (memcached_success(rc)) { if (trail != 0) { SAS::Event got_data(trail, SASEvent::MEMCACHED_GET_SUCCESS, 0); got_data.add_var_param(fqkey); got_data.add_var_param(data); got_data.add_static_param(cas); SAS::report_event(got_data); } // Return the data and CAS value. The CAS value is either set to the CAS // value from the result, or zero if an earlier active replica returned // NOT_FOUND. This ensures that a subsequent set operation will succeed // on the earlier active replica. LOG_DEBUG("Read %d bytes from table %s key %s, CAS = %ld", data.length(), table.c_str(), key.c_str(), cas); } else if (failed_replicas < replicas.size()) { // At least one replica returned NOT_FOUND. if (trail != 0) { SAS::Event not_found(trail, SASEvent::MEMCACHED_GET_NOT_FOUND, 0); not_found.add_var_param(fqkey); SAS::report_event(not_found); } LOG_DEBUG("At least one replica returned not found, so return NOT_FOUND"); status = Store::Status::NOT_FOUND; } else { // All replicas returned an error, so log the error and return the // failure. if (trail != 0) { SAS::Event err(trail, SASEvent::MEMCACHED_GET_ERROR, 0); err.add_var_param(fqkey); SAS::report_event(err); } LOG_ERROR("Failed to read data for %s from %d replicas", fqkey.c_str(), replicas.size()); status = Store::Status::ERROR; } return status; }
memcached_return_t memcached_purge(memcached_server_write_instance_st ptr) { uint32_t x; memcached_return_t ret= MEMCACHED_SUCCESS; memcached_st *root= (memcached_st *)ptr->root; uint32_t no_msg; if (memcached_is_purging(ptr->root) || /* already purging */ (memcached_server_response_count(ptr) < ptr->root->io_msg_watermark && ptr->io_bytes_sent < ptr->root->io_bytes_watermark) || (ptr->io_bytes_sent >= ptr->root->io_bytes_watermark && memcached_server_response_count(ptr) < 2)) { return MEMCACHED_SUCCESS; } /* memcached_io_write and memcached_response may call memcached_purge so we need to be able stop any recursion.. */ memcached_set_purging(root, true); WATCHPOINT_ASSERT(ptr->fd != -1); /* Force a flush of the buffer to ensure that we don't have the n-1 pending requests buffered up.. */ if (memcached_io_write(ptr, NULL, 0, true) == -1) { memcached_set_purging(root, true); return MEMCACHED_WRITE_FAILURE; } WATCHPOINT_ASSERT(ptr->fd != -1); no_msg= memcached_server_response_count(ptr) - 1; if (no_msg > 0) { memcached_result_st result; memcached_result_st *result_ptr; char buffer[SMALL_STRING_LEN]; /* * We need to increase the timeout, because we might be waiting for * data to be sent from the server (the commands was in the output buffer * and just flushed */ const int32_t timeo= ptr->root->poll_timeout; root->poll_timeout= 2000; result_ptr= memcached_result_create(root, &result); WATCHPOINT_ASSERT(result_ptr); for (x= 0; x < no_msg; x++) { memcached_return_t rc; memcached_result_reset(result_ptr); rc= memcached_read_one_response(ptr, buffer, sizeof (buffer), result_ptr); /* * Purge doesn't care for what kind of command results that is received. * The only kind of errors I care about if is I'm out of sync with the * protocol or have problems reading data from the network.. */ if (rc== MEMCACHED_PROTOCOL_ERROR || rc == MEMCACHED_UNKNOWN_READ_FAILURE) { WATCHPOINT_ERROR(rc); ret = rc; memcached_io_reset(ptr); } if (ptr->root->callbacks != NULL) { memcached_callback_st cb = *ptr->root->callbacks; if (rc == MEMCACHED_SUCCESS) { uint32_t y; for (y= 0; y < cb.number_of_callback; y++) { rc = (*cb.callback[y])(ptr->root, result_ptr, cb.context); if (rc != MEMCACHED_SUCCESS) break; } } } } memcached_result_free(result_ptr); root->poll_timeout= timeo; } memcached_set_purging(root, false); return ret; }