void ContextTransport::OnMessage(uint64_t nodeID, ReadBuffer msg)
{
int nread;
char proto;
Log_Trace("%R", &msg);
if (msg.GetLength() < 2)
ASSERT_FAIL();
nread = msg.Readf("%c:", &proto);
if (nread < 2)
ASSERT_FAIL();
msg.Advance(2);
switch (proto)
{
case PROTOCOL_CLUSTER:
OnClusterMessage(nodeID, msg);
break;
case PROTOCOL_QUORUM:
OnQuorumMessage(nodeID, msg);
break;
default:
ASSERT_FAIL();
break;
}
}
void ContextTransport::OnQuorumMessage(uint64_t nodeID, ReadBuffer& msg)
{
int nread;
uint64_t quorumID;
QuorumContext* quorumContext;
nread = msg.Readf("%U:", &quorumID);
if (nread < 2)
ASSERT_FAIL();
msg.Advance(nread);
quorumContext = GetQuorumContext(quorumID);
if (quorumContext)
GetQuorumContext(quorumID)->OnMessage(nodeID, msg);
}
bool ClusterConnection::OnMessage(ReadBuffer& msg)
{
uint64_t otherNodeID;
uint64_t otherClusterID;
ReadBuffer buffer;
ClusterConnection* dup;
int read;
//Log_Debug("ClusterConnection::OnMessage");
if (progress == ClusterConnection::INCOMING)
{
// we have no incoming connections if we don't have a nodeID
ASSERT(transport->IsAwaitingNodeID() == false);
// the node at the other end is awaiting its nodeID
if (msg.GetCharAt(0) == '*')
{
read = msg.Readf("*:%#R", &buffer);
if (read != (int) msg.GetLength())
{
// protocol error
GetSocket().GetEndpoint(endpoint);
Log_Message("[%s] Cluster protocol error, disconnecting...", endpoint.ToString());
transport->DeleteConnection(this);
return true;
}
if (!endpoint.Set(buffer, true))
{
Log_Message("[%R] Cluster invalid network address", &buffer);
transport->DeleteConnection(this);
return true;
}
progress = ClusterConnection::AWAITING_NODEID;
Log_Trace("Conn is awaiting nodeID at %s", endpoint.ToString());
transport->AddConnection(this);
if (transport->OnAwaitingNodeID(endpoint))
{
Log_Trace();
transport->DeleteConnection(this);
return true;
}
if (nodeID == UNDEFINED_NODEID)
Log_Message("[%s] Cluster unknown node connected <=", endpoint.ToString());
else
Log_Message("[%s] Cluster node %U connected <=", endpoint.ToString(), nodeID);
return false;
}
// both ends have nodeIDs
read = msg.Readf("%U:%U:%#R", &otherClusterID, &otherNodeID, &buffer);
if (read != (int) msg.GetLength())
{
// protocol error
GetSocket().GetEndpoint(endpoint);
Log_Message("[%s] Cluster protocol error, disconnecting...", endpoint.ToString());
transport->DeleteConnection(this);
return true;
}
if (otherClusterID > 0)
{
if (transport->GetClusterID() == 0)
{
// the other side has a clusterID, I don't, so set mine
// if I'm a config server:
// the clusterID also needs to be set in REPLICATON_CONFIG,
// this is set in ConfigServer::OnConnectionReady()
// if I'm a shard server:
// the controllers will send a SetNodeID message, which
// contains the clusterID, so I'll be fine
transport->SetClusterID(otherClusterID);
}
else
{
if (otherClusterID != transport->GetClusterID())
{
Log_Message("[%R] Cluster invalid configuration, disconnecting...", &buffer);
Log_Debug("mine: %U != controller %U", transport->GetClusterID(), otherClusterID);
transport->DeleteConnection(this); // drop this
return true;
}
}
}
dup = transport->GetConnection(otherNodeID);
if (dup)
{
// if the other connections isn't ready yet, drop it
// OR
// the other connection is ready
// in this case, kill the connection that was initiated by higher nodeID
// in other words, since this is an incoming connection:
// if nodeID[of initiator] > transport->GetSelfNodeID(): drop
if (dup->progress != READY || otherNodeID > transport->GetSelfNodeID())
{
Log_Trace("delete dup");
transport->DeleteConnection(dup); // drop dup
}
else if (otherNodeID != transport->GetSelfNodeID())
{
Log_Trace("delete this");
transport->DeleteConnection(this); // drop this
return true;
}
}
progress = ClusterConnection::READY;
SetNodeID(otherNodeID);
if (!endpoint.Set(buffer, true))
{
Log_Message("[%R] Cluster invalid network address", &buffer);
transport->DeleteConnection(this);
return true;
}
// check if the other side is not sending its localhost address, when they are on
// different nodes
if (endpoint.GetAddress() == Endpoint::GetLoopbackAddress() &&
transport->GetSelfEndpoint().GetAddress() != Endpoint::GetLoopbackAddress())
{
Log_Message("[%R] Cluster invalid network address", &buffer);
transport->DeleteConnection(this);
return true;
}
Log_Trace("Conn READY to node %U at %s", nodeID, endpoint.ToString());
if (nodeID != transport->GetSelfNodeID())
{
if (nodeID == UNDEFINED_NODEID)
Log_Message("[%s] Cluster unknown node connected <=", endpoint.ToString());
else
Log_Message("[%s] Cluster node %U connected <=", endpoint.ToString(), nodeID);
}
transport->AddConnection(this);
transport->OnWriteReadyness(this);
transport->OnConnectionReady(nodeID, endpoint);
}
else if (progress == ClusterConnection::OUTGOING)
ASSERT_FAIL();
else
transport->OnMessage(nodeID, msg); // pass msg to upper layer
return false;
}
bool ConfigMessage::Read(ReadBuffer& buffer)
{
int read;
unsigned numNodes, i;
uint64_t nodeID_;
ReadBuffer rb;
if (buffer.GetLength() < 1)
return false;
switch (buffer.GetCharAt(0))
{
// Cluster management
case CONFIGMESSAGE_SET_CLUSTER_ID:
read = buffer.Readf("%c:%U",
&type, &clusterID);
break;
case CONFIGMESSAGE_REGISTER_SHARDSERVER:
read = buffer.Readf("%c:%U:%#R",
&type, &nodeID, &rb);
endpoint.Set(rb);
break;
case CONFIGMESSAGE_UNREGISTER_SHARDSERVER:
read = buffer.Readf("%c:%U",
&type, &nodeID);
break;
case CONFIGMESSAGE_CREATE_QUORUM:
read = buffer.Readf("%c:%#R:%u",
&type, &name, &numNodes);
if (read < 0 || read == (signed)buffer.GetLength())
return false;
buffer.Advance(read);
for (i = 0; i < numNodes; i++)
{
read = buffer.Readf(":%U", &nodeID_);
if (read < 0)
return false;
buffer.Advance(read);
nodes.Append(nodeID_);
}
if (buffer.GetLength() == 0)
return true;
else
return false;
break;
case CONFIGMESSAGE_RENAME_QUORUM:
read = buffer.Readf("%c:%U:%#R",
&type, &quorumID, &name);
break;
case CONFIGMESSAGE_DELETE_QUORUM:
read = buffer.Readf("%c:%U",
&type, &quorumID);
break;
case CONFIGMESSAGE_ADD_SHARDSERVER_TO_QUORUM:
read = buffer.Readf("%c:%U:%U",
&type, &quorumID, &nodeID);
break;
case CONFIGMESSAGE_REMOVE_SHARDSERVER_FROM_QUORUM:
read = buffer.Readf("%c:%U:%U",
&type, &quorumID, &nodeID);
break;
case CONFIGMESSAGE_ACTIVATE_SHARDSERVER:
read = buffer.Readf("%c:%U:%U",
&type, &quorumID, &nodeID);
break;
case CONFIGMESSAGE_DEACTIVATE_SHARDSERVER:
read = buffer.Readf("%c:%U:%U",
&type, &quorumID, &nodeID);
break;
// Database management
case CONFIGMESSAGE_CREATE_DATABASE:
read = buffer.Readf("%c:%#R",
&type, &name);
break;
case CONFIGMESSAGE_RENAME_DATABASE:
read = buffer.Readf("%c:%U:%#R",
&type, &databaseID, &name);
break;
case CONFIGMESSAGE_DELETE_DATABASE:
read = buffer.Readf("%c:%U",
&type, &databaseID);
break;
// Table management
case CONFIGMESSAGE_CREATE_TABLE:
read = buffer.Readf("%c:%U:%U:%#R",
&type, &databaseID, &quorumID, &name);
break;
case CONFIGMESSAGE_RENAME_TABLE:
read = buffer.Readf("%c:%U:%#R",
&type, &tableID, &name);
break;
case CONFIGMESSAGE_DELETE_TABLE:
read = buffer.Readf("%c:%U",
&type, &tableID);
break;
case CONFIGMESSAGE_TRUNCATE_TABLE_BEGIN:
read = buffer.Readf("%c:%U",
&type, &tableID);
break;
case CONFIGMESSAGE_TRUNCATE_TABLE_COMPLETE:
read = buffer.Readf("%c:%U",
&type, &tableID);
break;
case CONFIGMESSAGE_FREEZE_TABLE:
read = buffer.Readf("%c:%U",
&type, &tableID);
break;
case CONFIGMESSAGE_UNFREEZE_TABLE:
read = buffer.Readf("%c:%U",
&type, &tableID);
break;
// Shard management
case CONFIGMESSAGE_SPLIT_SHARD_BEGIN:
read = buffer.Readf("%c:%U:%#B",
&type, &shardID, &splitKey);
break;
case CONFIGMESSAGE_SPLIT_SHARD_COMPLETE:
read = buffer.Readf("%c:%U",
&type, &shardID);
break;
case CONFIGMESSAGE_SHARD_MIGRATION_BEGIN:
read = buffer.Readf("%c:%U:%U",
&type, &quorumID, &shardID);
break;
case CONFIGMESSAGE_SHARD_MIGRATION_COMPLETE:
read = buffer.Readf("%c:%U:%U:%U",
&type, &quorumID, &srcShardID, &dstShardID);
break;
default:
return false;
}
return (read == (signed)buffer.GetLength() ? true : false);
}
bool PaxosMessage::Read(ReadBuffer& buffer)
{
int read;
char proto;
if (buffer.GetLength() < 3)
return false;
switch (buffer.GetCharAt(2))
{
case PAXOS_PREPARE_REQUEST:
read = buffer.Readf("%c:%c:%U:%U:%U",
&proto, &type, &paxosID, &nodeID, &proposalID);
break;
case PAXOS_PREPARE_REJECTED:
read = buffer.Readf("%c:%c:%U:%U:%U:%U",
&proto, &type, &paxosID, &nodeID, &proposalID, &promisedProposalID);
break;
case PAXOS_PREPARE_PREVIOUSLY_ACCEPTED:
read = buffer.Readf("%c:%c:%U:%U:%U:%U:%U:%#R",
&proto, &type, &paxosID, &nodeID, &proposalID, &acceptedProposalID, &runID, &value);
break;
case PAXOS_PREPARE_CURRENTLY_OPEN:
read = buffer.Readf("%c:%c:%U:%U:%U",
&proto, &type, &paxosID, &nodeID, &proposalID);
break;
case PAXOS_PROPOSE_REQUEST:
read = buffer.Readf("%c:%c:%U:%U:%U:%U:%#R",
&proto, &type, &paxosID, &nodeID, &proposalID, &runID, &value);
break;
case PAXOS_PROPOSE_REJECTED:
read = buffer.Readf("%c:%c:%U:%U:%U",
&proto, &type, &paxosID, &nodeID, &proposalID);
break;
case PAXOS_PROPOSE_ACCEPTED:
read = buffer.Readf("%c:%c:%U:%U:%U",
&proto, &type, &paxosID, &nodeID, &proposalID);
break;
case PAXOS_LEARN_PROPOSAL:
read = buffer.Readf("%c:%c:%U:%U:%U",
&proto, &type, &paxosID, &nodeID, &proposalID);
break;
case PAXOS_LEARN_VALUE:
read = buffer.Readf("%c:%c:%U:%U:%U:%#R",
&proto, &type, &paxosID, &nodeID, &runID, &value);
break;
case PAXOS_REQUEST_CHOSEN:
read = buffer.Readf("%c:%c:%U:%U",
&proto, &type, &paxosID, &nodeID);
break;
case PAXOS_START_CATCHUP:
read = buffer.Readf("%c:%c:%U:%U",
&proto, &type, &paxosID, &nodeID);
break;
default:
return false;
}
ASSERT(proto == PAXOS_PROTOCOL_ID);
return (read == (signed)buffer.GetLength());
}
bool SDBPRequestMessage::Read(ReadBuffer& buffer)
{
int read;
unsigned i, numNodes;
uint64_t nodeID;
ReadBuffer optional;
if (buffer.GetLength() < 1)
return false;
switch (buffer.GetCharAt(0))
{
/* Master query */
case CLIENTREQUEST_GET_MASTER:
read = buffer.Readf("%c:%U",
&request->type, &request->commandID);
break;
/* Get config state: databases, tables, shards, quora */
case CLIENTREQUEST_GET_CONFIG_STATE:
read = buffer.Readf("%c:%U",
&request->type, &request->commandID);
break;
/* Quorum management */
case CLIENTREQUEST_CREATE_QUORUM:
read = buffer.Readf("%c:%U:%u",
&request->type, &request->commandID, &numNodes);
if (read < 0 || read == (signed)buffer.GetLength())
return false;
buffer.Advance(read);
for (i = 0; i < numNodes; i++)
{
read = buffer.Readf(":%U", &nodeID);
if (read < 0)
return false;
buffer.Advance(read);
request->nodes.Append(nodeID);
}
if (buffer.GetLength() == 0)
return true;
else
return false;
break;
case CLIENTREQUEST_DELETE_QUORUM:
read = buffer.Readf("%c:%U:%U",
&request->type, &request->commandID, &request->quorumID);
break;
case CLIENTREQUEST_ADD_NODE:
read = buffer.Readf("%c:%U:%U:%U",
&request->type, &request->commandID, &request->quorumID, &request->nodeID);
break;
case CLIENTREQUEST_REMOVE_NODE:
read = buffer.Readf("%c:%U:%U:%U",
&request->type, &request->commandID, &request->quorumID, &request->nodeID);
break;
case CLIENTREQUEST_ACTIVATE_NODE:
read = buffer.Readf("%c:%U:%U",
&request->type, &request->commandID, &request->nodeID);
break;
/* Database management */
case CLIENTREQUEST_CREATE_DATABASE:
read = buffer.Readf("%c:%U:%#B",
&request->type, &request->commandID, &request->name);
break;
case CLIENTREQUEST_RENAME_DATABASE:
read = buffer.Readf("%c:%U:%U:%#B",
&request->type, &request->commandID, &request->databaseID,
&request->name);
break;
case CLIENTREQUEST_DELETE_DATABASE:
read = buffer.Readf("%c:%U:%U",
&request->type, &request->commandID, &request->databaseID);
break;
case CLIENTREQUEST_SPLIT_SHARD:
read = buffer.Readf("%c:%U:%U:%#B",
&request->type, &request->commandID,
&request->shardID, &request->key);
break;
case CLIENTREQUEST_MIGRATE_SHARD:
read = buffer.Readf("%c:%U:%U:%U",
&request->type, &request->commandID, &request->shardID, &request->quorumID);
return true;
/* Table management */
case CLIENTREQUEST_CREATE_TABLE:
read = buffer.Readf("%c:%U:%U:%U:%#B",
&request->type, &request->commandID, &request->databaseID,
&request->quorumID, &request->name);
break;
case CLIENTREQUEST_RENAME_TABLE:
read = buffer.Readf("%c:%U:%U:%#B",
&request->type, &request->commandID,
&request->tableID, &request->name);
break;
case CLIENTREQUEST_DELETE_TABLE:
read = buffer.Readf("%c:%U:%U",
&request->type, &request->commandID,
&request->tableID);
break;
case CLIENTREQUEST_TRUNCATE_TABLE:
read = buffer.Readf("%c:%U:%U",
&request->type, &request->commandID,
&request->tableID);
break;
/* Data operations */
case CLIENTREQUEST_GET:
read = buffer.Readf("%c:%U:%U:%U:%#B",
&request->type, &request->commandID,
&request->tableID, &request->paxosID,
&request->key);
break;
case CLIENTREQUEST_SET:
case CLIENTREQUEST_SET_IF_NOT_EXISTS:
case CLIENTREQUEST_GET_AND_SET:
read = buffer.Readf("%c:%U:%U:%#B:%#B",
&request->type, &request->commandID,
&request->tableID, &request->key, &request->value);
break;
case CLIENTREQUEST_TEST_AND_SET:
read = buffer.Readf("%c:%U:%U:%#B:%#B:%#B",
&request->type, &request->commandID,
&request->tableID, &request->key, &request->test, &request->value);
break;
case CLIENTREQUEST_TEST_AND_DELETE:
read = buffer.Readf("%c:%U:%U:%#B:%#B",
&request->type, &request->commandID,
&request->tableID, &request->key, &request->test);
break;
case CLIENTREQUEST_ADD:
read = buffer.Readf("%c:%U:%U:%#B:%I",
&request->type, &request->commandID,
&request->tableID, &request->key, &request->number);
break;
case CLIENTREQUEST_APPEND:
read = buffer.Readf("%c:%U:%U:%#B:%#B",
&request->type, &request->commandID,
&request->tableID, &request->key, &request->value);
break;
case CLIENTREQUEST_DELETE:
case CLIENTREQUEST_REMOVE:
read = buffer.Readf("%c:%U:%U:%#B",
&request->type, &request->commandID,
&request->tableID, &request->key);
break;
case CLIENTREQUEST_LIST_KEYS:
case CLIENTREQUEST_LIST_KEYVALUES:
case CLIENTREQUEST_COUNT:
read = buffer.Readf("%c:%U:%U:%#B:%#B:%#B:%U:%U",
&request->type, &request->commandID,
&request->tableID, &request->key, &request->endKey, &request->prefix,
&request->count, &request->offset);
break;
case CLIENTREQUEST_SUBMIT:
read = buffer.Readf("%c:%U",
&request->type, &request->quorumID);
break;
case CLIENTREQUEST_BULK_LOADING:
read = buffer.Readf("%c:%U",
&request->type, &request->commandID);
break;
default:
return false;
}
return (read == (signed)buffer.GetLength() ? true : false);
}
bool ClusterMessage::Read(ReadBuffer& buffer)
{
#define READ_SEPARATOR() \
read = buffer.Readf(":"); \
if (read != 1) \
return false; \
buffer.Advance(read); \
int read;
ReadBuffer tempBuffer;
if (buffer.GetLength() < 1)
return false;
switch (buffer.GetCharAt(0))
{
case CLUSTERMESSAGE_SET_NODEID:
read = buffer.Readf("%c:%U:%U",
&type, &clusterID, &nodeID);
break;
case CLUSTERMESSAGE_UNREGISTER_STOP:
read = buffer.Readf("%c",
&type);
break;
case CLUSTERMESSAGE_HEARTBEAT:
read = buffer.Readf("%c:%U:%u:%u",
&type, &nodeID, &httpPort, &sdbpPort);
if (read < 3)
return false;
buffer.Advance(read);
READ_SEPARATOR();
if (!QuorumInfo::ReadList(buffer, quorumInfos))
return false;
READ_SEPARATOR();
if (!QuorumShardInfo::ReadList(buffer, quorumShardInfos))
return false;
return true;
break;
case CLUSTERMESSAGE_SET_CONFIG_STATE:
type = CLUSTERMESSAGE_SET_CONFIG_STATE;
return configState.Read(buffer, true);
case CLUSTERMESSAGE_REQUEST_LEASE:
read = buffer.Readf("%c:%U:%U:%U:%U:%U:%u",
&type, &nodeID, &quorumID, &proposalID, &paxosID, &configID, &duration);
break;
case CLUSTERMESSAGE_RECEIVE_LEASE:
read = buffer.Readf("%c:%U:%U:%U:%U:%u:%b",
&type, &nodeID, &quorumID, &proposalID, &configID, &duration, &watchingPaxosID);
if (read < 9)
return false;
buffer.Advance(read);
READ_SEPARATOR();
if (!MessageUtil::ReadIDList(activeNodes, buffer))
return false;
READ_SEPARATOR();
if (!MessageUtil::ReadIDList(shards, buffer))
return false;
return true;
case CLUSTERMESSAGE_SHARDMIGRATION_INITIATE:
read = buffer.Readf("%c:%U:%U:%U:%U",
&type, &nodeID, &quorumID, &srcShardID, &dstShardID);
break;
case CLUSTERMESSAGE_SHARDMIGRATION_BEGIN:
read = buffer.Readf("%c:%U:%U:%U",
&type, &quorumID, &srcShardID, &dstShardID);
break;
case CLUSTERMESSAGE_SHARDMIGRATION_SET:
read = buffer.Readf("%c:%U:%U:%#R:%#R",
&type, &quorumID, &shardID, &key, &value);
break;
case CLUSTERMESSAGE_SHARDMIGRATION_DELETE:
read = buffer.Readf("%c:%U:%U:%#R",
&type, &quorumID, &shardID, &key);
break;
case CLUSTERMESSAGE_SHARDMIGRATION_COMMIT:
read = buffer.Readf("%c:%U:%U",
&type, &quorumID, &shardID);
break;
case CLUSTERMESSAGE_SHARDMIGRATION_COMPLETE:
read = buffer.Readf("%c:%U:%U",
&type, &quorumID, &shardID);
break;
case CLUSTERMESSAGE_SHARDMIGRATION_PAUSE:
read = buffer.Readf("%c",
&type);
break;
case CLUSTERMESSAGE_SHARDMIGRATION_RESUME:
read = buffer.Readf("%c",
&type);
break;
case CLUSTERMESSAGE_HELLO:
read = buffer.Readf("%c:%U:%#R",
&type, &clusterID, &value);
break;
case CLUSTERMESSAGE_HTTP_ENDPOINT:
read = buffer.Readf("%c:%U:%#R",
&type, &nodeID, &endpoint);
break;
default:
return false;
}
return (read == (signed)buffer.GetLength());
#undef READ_SEPARATOR
}