/*
make a remote ctdb call - async send. Called in daemon context.
This constructs a ctdb_call request and queues it for processing.
This call never blocks.
*/
struct ctdb_call_state *ctdb_daemon_call_send_remote(struct ctdb_db_context *ctdb_db,
struct ctdb_call *call,
struct ctdb_ltdb_header *header)
{
uint32_t len;
struct ctdb_call_state *state;
struct ctdb_context *ctdb = ctdb_db->ctdb;
state = talloc_zero(ctdb_db, struct ctdb_call_state);
CTDB_NO_MEMORY_NULL(ctdb, state);
len = offsetof(struct ctdb_req_call, data) + call->key.dsize + call->call_data.dsize;
state->c = ctdb->methods->allocate_pkt(state, len);
CTDB_NO_MEMORY_NULL(ctdb, state->c);
talloc_set_name_const(state->c, "req_call packet");
state->c->hdr.length = len;
state->c->hdr.ctdb_magic = CTDB_MAGIC;
state->c->hdr.ctdb_version = CTDB_VERSION;
state->c->hdr.operation = CTDB_REQ_CALL;
state->c->hdr.destnode = header->dmaster;
state->c->hdr.srcnode = ctdb->vnn;
/* this limits us to 16k outstanding messages - not unreasonable */
state->c->hdr.reqid = idr_get_new(ctdb->idr, state, 0xFFFF);
state->c->flags = call->flags;
state->c->db_id = ctdb_db->db_id;
state->c->callid = call->call_id;
state->c->keylen = call->key.dsize;
state->c->calldatalen = call->call_data.dsize;
memcpy(&state->c->data[0], call->key.dptr, call->key.dsize);
memcpy(&state->c->data[call->key.dsize],
call->call_data.dptr, call->call_data.dsize);
state->call = *call;
state->call.call_data.dptr = &state->c->data[call->key.dsize];
state->call.key.dptr = &state->c->data[0];
state->node = ctdb->nodes[header->dmaster];
state->state = CTDB_CALL_WAIT;
state->header = *header;
state->ctdb_db = ctdb_db;
talloc_set_destructor(state, ctdb_call_destructor);
ctdb_queue_packet(ctdb, &state->c->hdr);
event_add_timed(ctdb->ev, state, timeval_current_ofs(CTDB_REQ_TIMEOUT, 0),
ctdb_call_timeout, state);
return state;
}
/*
called when a CTDB_REPLY_REDIRECT packet comes in
This packet arrives when we have sent a CTDB_REQ_CALL request and
the node that received it is not the dmaster for the given key. We
are given a hint as to what node to try next.
*/
void ctdb_reply_redirect(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
{
struct ctdb_reply_redirect *c = (struct ctdb_reply_redirect *)hdr;
struct ctdb_call_state *state;
state = idr_find_type(ctdb->idr, hdr->reqid, struct ctdb_call_state);
if (state == NULL) return;
talloc_steal(state, c);
/* don't allow for too many redirects */
if (state->redirect_count++ == CTDB_MAX_REDIRECT) {
c->dmaster = ctdb_lmaster(ctdb, &state->call.key);
}
/* send it off again */
state->node = ctdb->nodes[c->dmaster];
state->c->hdr.destnode = c->dmaster;
ctdb_queue_packet(ctdb, &state->c->hdr);
}
/*
send a redirect reply
*/
static void ctdb_call_send_redirect(struct ctdb_context *ctdb,
struct ctdb_req_call *c,
struct ctdb_ltdb_header *header)
{
struct ctdb_reply_redirect *r;
r = ctdb->methods->allocate_pkt(ctdb, sizeof(*r));
CTDB_NO_MEMORY_FATAL(ctdb, r);
talloc_set_name_const(r, "send_redirect packet");
r->hdr.length = sizeof(*r);
r->hdr.ctdb_magic = CTDB_MAGIC;
r->hdr.ctdb_version = CTDB_VERSION;
r->hdr.operation = CTDB_REPLY_REDIRECT;
r->hdr.destnode = c->hdr.srcnode;
r->hdr.srcnode = ctdb->vnn;
r->hdr.reqid = c->hdr.reqid;
r->dmaster = header->dmaster;
ctdb_queue_packet(ctdb, &r->hdr);
talloc_free(r);
}
static void ctdb_send_error(struct ctdb_context *ctdb,
struct ctdb_req_header *hdr, uint32_t status,
const char *fmt, ...)
{
va_list ap;
struct ctdb_reply_error *r;
char *msg;
int msglen, len;
va_start(ap, fmt);
msg = talloc_vasprintf(ctdb, fmt, ap);
if (msg == NULL) {
ctdb_fatal(ctdb, "Unable to allocate error in ctdb_send_error\n");
}
va_end(ap);
msglen = strlen(msg)+1;
len = offsetof(struct ctdb_reply_error, msg);
r = ctdb->methods->allocate_pkt(msg, len + msglen);
CTDB_NO_MEMORY_FATAL(ctdb, r);
talloc_set_name_const(r, "send_error packet");
r->hdr.length = len + msglen;
r->hdr.ctdb_magic = CTDB_MAGIC;
r->hdr.ctdb_version = CTDB_VERSION;
r->hdr.operation = CTDB_REPLY_ERROR;
r->hdr.destnode = hdr->srcnode;
r->hdr.srcnode = ctdb->vnn;
r->hdr.reqid = hdr->reqid;
r->status = status;
r->msglen = msglen;
memcpy(&r->msg[0], msg, msglen);
ctdb_queue_packet(ctdb, &r->hdr);
talloc_free(msg);
}
/*
send a dmaster request (give another node the dmaster for a record)
This is always sent to the lmaster, which ensures that the lmaster
always knows who the dmaster is. The lmaster will then send a
CTDB_REPLY_DMASTER to the new dmaster
*/
static void ctdb_call_send_dmaster(struct ctdb_db_context *ctdb_db,
struct ctdb_req_call *c,
struct ctdb_ltdb_header *header,
TDB_DATA *key, TDB_DATA *data)
{
struct ctdb_req_dmaster *r;
struct ctdb_context *ctdb = ctdb_db->ctdb;
int len;
len = offsetof(struct ctdb_req_dmaster, data) + key->dsize + data->dsize;
r = ctdb->methods->allocate_pkt(ctdb, len);
CTDB_NO_MEMORY_FATAL(ctdb, r);
talloc_set_name_const(r, "send_dmaster packet");
r->hdr.length = len;
r->hdr.ctdb_magic = CTDB_MAGIC;
r->hdr.ctdb_version = CTDB_VERSION;
r->hdr.operation = CTDB_REQ_DMASTER;
r->hdr.destnode = ctdb_lmaster(ctdb, key);
r->hdr.srcnode = ctdb->vnn;
r->hdr.reqid = c->hdr.reqid;
r->db_id = c->db_id;
r->dmaster = c->hdr.srcnode;
r->keylen = key->dsize;
r->datalen = data->dsize;
memcpy(&r->data[0], key->dptr, key->dsize);
memcpy(&r->data[key->dsize], data->dptr, data->dsize);
/* XXX - probably not necessary when lmaster==dmaster
update the ltdb to record the new dmaster */
header->dmaster = r->hdr.destnode;
ctdb_ltdb_store(ctdb_db, *key, header, *data);
ctdb_queue_packet(ctdb, &r->hdr);
talloc_free(r);
}
/*
called when a CTDB_REQ_CALL packet comes in
*/
void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
{
struct ctdb_req_call *c = (struct ctdb_req_call *)hdr;
TDB_DATA data;
struct ctdb_reply_call *r;
int ret, len;
struct ctdb_ltdb_header header;
struct ctdb_call call;
struct ctdb_db_context *ctdb_db;
ctdb_db = find_ctdb_db(ctdb, c->db_id);
if (!ctdb_db) {
ctdb_send_error(ctdb, hdr, -1,
"Unknown database in request. db_id==0x%08x",
c->db_id);
return;
}
call.call_id = c->callid;
call.key.dptr = c->data;
call.key.dsize = c->keylen;
call.call_data.dptr = c->data + c->keylen;
call.call_data.dsize = c->calldatalen;
/* determine if we are the dmaster for this key. This also
fetches the record data (if any), thus avoiding a 2nd fetch of the data
if the call will be answered locally */
ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, call.key, &header, hdr, &data,
ctdb_recv_raw_pkt, ctdb);
if (ret == -1) {
ctdb_send_error(ctdb, hdr, ret, "ltdb fetch failed in ctdb_request_call");
return;
}
if (ret == -2) {
DEBUG(2,(__location__ " deferred ctdb_request_call\n"));
return;
}
/* if we are not the dmaster, then send a redirect to the
requesting node */
if (header.dmaster != ctdb->vnn) {
ctdb_call_send_redirect(ctdb, c, &header);
talloc_free(data.dptr);
ctdb_ltdb_unlock(ctdb_db, call.key);
return;
}
/* if this nodes has done enough consecutive calls on the same record
then give them the record
or if the node requested an immediate migration
*/
if ( (header.laccessor == c->hdr.srcnode
&& header.lacount >= ctdb->max_lacount)
|| c->flags&CTDB_IMMEDIATE_MIGRATION ) {
ctdb_call_send_dmaster(ctdb_db, c, &header, &call.key, &data);
talloc_free(data.dptr);
ctdb_ltdb_unlock(ctdb_db, call.key);
return;
}
ctdb_call_local(ctdb_db, &call, &header, &data, c->hdr.srcnode);
ctdb_ltdb_unlock(ctdb_db, call.key);
len = offsetof(struct ctdb_reply_call, data) + call.reply_data.dsize;
r = ctdb->methods->allocate_pkt(ctdb, len);
CTDB_NO_MEMORY_FATAL(ctdb, r);
talloc_set_name_const(r, "reply_call packet");
r->hdr.length = len;
r->hdr.ctdb_magic = CTDB_MAGIC;
r->hdr.ctdb_version = CTDB_VERSION;
r->hdr.operation = CTDB_REPLY_CALL;
r->hdr.destnode = hdr->srcnode;
r->hdr.srcnode = hdr->destnode;
r->hdr.reqid = hdr->reqid;
r->status = call.status;
r->datalen = call.reply_data.dsize;
if (call.reply_data.dsize) {
memcpy(&r->data[0], call.reply_data.dptr, call.reply_data.dsize);
talloc_free(call.reply_data.dptr);
}
ctdb_queue_packet(ctdb, &r->hdr);
talloc_free(r);
}
/*
called when a CTDB_REQ_DMASTER packet comes in
this comes into the lmaster for a record when the current dmaster
wants to give up the dmaster role and give it to someone else
*/
void ctdb_request_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
{
struct ctdb_req_dmaster *c = (struct ctdb_req_dmaster *)hdr;
struct ctdb_reply_dmaster *r;
TDB_DATA key, data, data2;
struct ctdb_ltdb_header header;
struct ctdb_db_context *ctdb_db;
int ret, len;
TALLOC_CTX *tmp_ctx;
key.dptr = c->data;
key.dsize = c->keylen;
data.dptr = c->data + c->keylen;
data.dsize = c->datalen;
ctdb_db = find_ctdb_db(ctdb, c->db_id);
if (!ctdb_db) {
ctdb_send_error(ctdb, hdr, -1,
"Unknown database in request. db_id==0x%08x",
c->db_id);
return;
}
/* fetch the current record */
ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, key, &header, hdr, &data2,
ctdb_recv_raw_pkt, ctdb);
if (ret == -1) {
ctdb_fatal(ctdb, "ctdb_req_dmaster failed to fetch record");
return;
}
if (ret == -2) {
DEBUG(2,(__location__ " deferring ctdb_request_dmaster\n"));
return;
}
/* its a protocol error if the sending node is not the current dmaster */
if (header.dmaster != hdr->srcnode &&
hdr->srcnode != ctdb_lmaster(ctdb_db->ctdb, &key)) {
ctdb_fatal(ctdb, "dmaster request from non-master");
return;
}
header.dmaster = c->dmaster;
ret = ctdb_ltdb_store(ctdb_db, key, &header, data);
ctdb_ltdb_unlock(ctdb_db, key);
if (ret != 0) {
ctdb_fatal(ctdb, "ctdb_req_dmaster unable to update dmaster");
return;
}
/* put the packet on a temporary context, allowing us to safely free
it below even if ctdb_reply_dmaster() has freed it already */
tmp_ctx = talloc_new(ctdb);
/* send the CTDB_REPLY_DMASTER */
len = offsetof(struct ctdb_reply_dmaster, data) + data.dsize;
r = ctdb->methods->allocate_pkt(tmp_ctx, len);
CTDB_NO_MEMORY_FATAL(ctdb, r);
talloc_set_name_const(r, "reply_dmaster packet");
r->hdr.length = len;
r->hdr.ctdb_magic = CTDB_MAGIC;
r->hdr.ctdb_version = CTDB_VERSION;
r->hdr.operation = CTDB_REPLY_DMASTER;
r->hdr.destnode = c->dmaster;
r->hdr.srcnode = ctdb->vnn;
r->hdr.reqid = hdr->reqid;
r->datalen = data.dsize;
memcpy(&r->data[0], data.dptr, data.dsize);
ctdb_queue_packet(ctdb, &r->hdr);
talloc_free(tmp_ctx);
}
/*
send a control message to a node
*/
int ctdb_daemon_send_control(struct ctdb_context *ctdb, uint32_t destnode,
uint64_t srvid, uint32_t opcode, uint32_t client_id,
uint32_t flags,
TDB_DATA data,
ctdb_control_callback_fn_t callback,
void *private_data)
{
struct ctdb_req_control_old *c;
struct ctdb_control_state *state;
size_t len;
if (ctdb->methods == NULL) {
DEBUG(DEBUG_INFO,(__location__ " Failed to send control. Transport is DOWN\n"));
return -1;
}
if (((destnode == CTDB_BROADCAST_VNNMAP) ||
(destnode == CTDB_BROADCAST_ALL) ||
(destnode == CTDB_BROADCAST_CONNECTED)) &&
!(flags & CTDB_CTRL_FLAG_NOREPLY)) {
DEBUG(DEBUG_CRIT,("Attempt to broadcast control without NOREPLY\n"));
return -1;
}
if (destnode != CTDB_BROADCAST_VNNMAP &&
destnode != CTDB_BROADCAST_ALL &&
destnode != CTDB_BROADCAST_CONNECTED &&
(!ctdb_validate_pnn(ctdb, destnode) ||
(ctdb->nodes[destnode]->flags & NODE_FLAGS_DISCONNECTED))) {
if (!(flags & CTDB_CTRL_FLAG_NOREPLY)) {
callback(ctdb, -1, tdb_null, "ctdb_control to disconnected node", private_data);
}
return 0;
}
/* the state is made a child of private_data if possible. This means any reply
will be discarded if the private_data goes away */
state = talloc(private_data?private_data:ctdb, struct ctdb_control_state);
CTDB_NO_MEMORY(ctdb, state);
state->reqid = reqid_new(ctdb->idr, state);
state->callback = callback;
state->private_data = private_data;
state->ctdb = ctdb;
state->flags = flags;
talloc_set_destructor(state, ctdb_control_destructor);
len = offsetof(struct ctdb_req_control_old, data) + data.dsize;
c = ctdb_transport_allocate(ctdb, state, CTDB_REQ_CONTROL, len,
struct ctdb_req_control_old);
CTDB_NO_MEMORY(ctdb, c);
talloc_set_name_const(c, "ctdb_req_control packet");
c->hdr.destnode = destnode;
c->hdr.reqid = state->reqid;
c->opcode = opcode;
c->client_id = client_id;
c->flags = flags;
c->srvid = srvid;
c->datalen = data.dsize;
if (data.dsize) {
memcpy(&c->data[0], data.dptr, data.dsize);
}
ctdb_queue_packet(ctdb, &c->hdr);
if (flags & CTDB_CTRL_FLAG_NOREPLY) {
talloc_free(state);
return 0;
}
if (ctdb->tunable.control_timeout) {
tevent_add_timer(ctdb->ev, state,
timeval_current_ofs(ctdb->tunable.control_timeout, 0),
ctdb_control_timeout, state);
}
talloc_free(c);
return 0;
}
