void ReadReadyHandler_MsgServer(AsyncCommBuffer buffer)
{
int res = FUNC_RETURN_OK;
ConnectionTrack newtrack = NULL;
ConnectionTrackData tmptrackdata;
AsyncCommBuffer newcommbuffer = NULL;
struct sockaddr_in clientaddr;
socklen_t clientaddrlen = sizeof(clientaddr);
int fd = -1;
/* Always accept the connection. */
fd = accept(buffer->FD, (struct sockaddr *)&clientaddr, &clientaddrlen);
if ( fd == -1 )
{
elog(WARNING, "Resource manager socket accept error is detected. errno %d",
errno);
return;
}
/* Create AsyncComm Message handler instance. */
AsyncCommMessageHandlerContext context = createConnTrackHandlerContext();
/* Add new client fd into AsyncComm manager. */
res = registerFileDesc(fd,
ASYNCCOMM_READBYTES | ASYNCCOMM_WRITEBYTES,
&AsyncCommBufferHandlersMessage,
context,
&newcommbuffer);
if ( res != FUNC_RETURN_OK )
{
Assert(newcommbuffer == NULL);
/* close the connection and cleanup. */
closeConnectionRemote(&fd);
rm_pfree(AsyncCommContext, context);
return;
}
assignFileDescClientAddressInfo(newcommbuffer,
NULL,
0,
&clientaddr,
clientaddrlen);
/* Let context able to track comm buffer. */
context->AsyncBuffer = newcommbuffer;
/* Call callback function to initialize context for message handlers. */
InitHandler_Message(newcommbuffer);
elog(DEBUG3, "Resource manager accepted one client connected from "
"%s:%d as FD %d.\n",
newcommbuffer->ClientAddrDotStr,
newcommbuffer->ClientAddrPort,
newcommbuffer->FD);
}
int initializeSocketServer_RMSEG(void)
{
int res = FUNC_RETURN_OK;
int netres = 0;
char *allip = "0.0.0.0";
pgsocket RMListenSocket[HAWQRM_SERVER_PORT_COUNT];
for ( int i = 0 ; i < HAWQRM_SERVER_PORT_COUNT ; ++i ) {
RMListenSocket[i] = PGINVALID_SOCKET;
}
/* Listen normal socket addresses. */
netres = StreamServerPort( AF_UNSPEC,
allip,
rm_segment_port,
NULL,
RMListenSocket,
HAWQRM_SERVER_PORT_COUNT);
/* If there are multiple segments in one host, which is common in old imp.
* We can not make all segments work. So, if HAWQ RM SEG fails to start
* socket server by listening the port, we accept this case and make it
* silent. HAWQ RM will not recognize this segment and will not assign
* tasks. */
if ( netres != STATUS_OK ) {
res = REQUESTHANDLER_FAIL_START_SOCKET_SERVER;
elog( LOG, "Can not create socket server. HostName=%s, Port=%d",
allip,
rm_segment_port);
return res;
}
/* Initialize array for polling all file descriptors. */
initializeAsyncComm();
int validfdcount = 0;
AsyncCommBuffer newbuffer = NULL;
for ( int i = 0 ; i < HAWQRM_SERVER_PORT_COUNT ; ++i ) {
if (RMListenSocket[i] != PGINVALID_SOCKET) {
netres = registerFileDesc(RMListenSocket[i],
NULL,
ASYNCCOMM_READ,
&AsyncCommBufferHandlersMsgServer,
NULL,
&newbuffer);
if ( netres != FUNC_RETURN_OK ) {
res = REQUESTHANDLER_FAIL_START_SOCKET_SERVER;
elog(LOG, "Resource manager cannot track socket server.");
break;
}
validfdcount++;
InitHandler_Message(newbuffer);
}
}
if ( res != FUNC_RETURN_OK ) {
for ( int i = 0 ; i < HAWQRM_SERVER_PORT_COUNT ; ++i ) {
if ( RMListenSocket[i] != PGINVALID_SOCKET ) close(RMListenSocket[i]);
}
return res;
}
elog(DEBUG5, "HAWQ RM SEG :: Starts accepting resource request. "
"Listening normal socket port %s:%d. "
"Total listened %d FDs.",
allip,
rm_segment_port,
validfdcount);
return res;
}
int sendRUAlive(char *seghostname)
{
int res = FUNC_RETURN_OK;
AsyncCommBuffer newcommbuffer = NULL;
SegResource segres = NULL;
int32_t segid = SEGSTAT_ID_INVALID;
Assert(seghostname != NULL);
res = getSegIDByHostName(seghostname, strlen(seghostname), &segid);
if ( res != FUNC_RETURN_OK )
{
elog(WARNING, "Resource manager cannot get registered host %s when send RUAlive message, "
"error code: %d",
seghostname,
res);
return res;
}
segres = getSegResource(segid);
Assert(segres != NULL);
bool oldstat = setSegResRUAlivePending(segres, true);
if( oldstat != false )
{
Assert(false);
}
/* Build request. */
SelfMaintainBufferData tosend;
initializeSelfMaintainBuffer(&tosend, PCONTEXT);
RPCRequestRUAliveData request;
request.Reserved = 0;
appendSMBVar(&tosend, request);
AsyncCommMessageHandlerContext context =
(AsyncCommMessageHandlerContext)
rm_palloc0(AsyncCommContext,
sizeof(AsyncCommMessageHandlerContextData));
context->inMessage = false;
context->MessageRecvReadyHandler = NULL;
context->MessageRecvedHandler = receivedRUAliveResponse;
context->MessageSendReadyHandler = NULL;
context->MessageSentHandler = sentRUAlive;
context->MessageErrorHandler = sentRUAliveError;
context->MessageCleanUpHandler = sentRUAliveCleanUp;
context->UserData = (void *)segres;
/* Connect to HAWQ RM server */
res = registerAsyncConnectionFileDesc(NULL,
seghostname,
rm_segment_port,
ASYNCCOMM_READBYTES | ASYNCCOMM_WRITEBYTES,
&AsyncCommBufferHandlersMessage,
context,
&newcommbuffer);
if ( res != FUNC_RETURN_OK )
{
rm_pfree(AsyncCommContext, context);
elog(WARNING, "Fail to register asynchronous connection for sending "
"RUAlive message. %d", res);
return res;
}
buildMessageToCommBuffer(newcommbuffer,
tosend.Buffer,
tosend.Cursor+1,
REQUEST_RM_RUALIVE,
0,
0);
destroySelfMaintainBuffer(&tosend);
context->AsyncBuffer = newcommbuffer;
InitHandler_Message(newcommbuffer);
return FUNC_RETURN_OK;
}
int decreaseMemoryQuota(char *seghostname,
GRMContainerSet rescontainerset)
{
int res = FUNC_RETURN_OK;
AsyncCommBuffer commbuffer = NULL;
GRMContainerSet ctns = NULL;
Assert( seghostname != NULL );
Assert( rescontainerset->Containers != NULL );
/* Move resource containers that should be returned to communication context.*/
ctns = createGRMContainerSet();
moveGRMContainerSetContainerList(ctns, rescontainerset);
/* Build request */
SelfMaintainBufferData tosend;
initializeSelfMaintainBuffer(&tosend, PCONTEXT);
RPCRequestUpdateMemoryQuotaData request;
request.MemoryQuotaDelta = ctns->Allocated.MemoryMB;
GRMContainer firstcont = getGRMContainerSetContainerFirst(ctns);
Assert( firstcont != NULL );
request.MemoryQuotaTotalPending = firstcont->Resource->Allocated.MemoryMB +
firstcont->Resource->OldInuse.MemoryMB;
Assert( request.MemoryQuotaTotalPending >= 0 );
elog(LOG, "Resource manager decrease host %s memory quota "UINT64_FORMAT" MB, "
"Expected total memory quota after decreasing is " UINT64_FORMAT" MB. "
"Include %d MB old in-use memory quota.",
seghostname,
request.MemoryQuotaDelta,
request.MemoryQuotaTotalPending,
firstcont->Resource->OldInuse.MemoryMB);
appendSMBVar(&tosend, request);
/* Set content to send and add to AsyncComm framework */
AsyncCommMessageHandlerContext context =
(AsyncCommMessageHandlerContext)rm_palloc0(AsyncCommContext,
sizeof(AsyncCommMessageHandlerContextData));
context->inMessage = false;
context->UserData = ctns;
context->MessageRecvReadyHandler = NULL;
context->MessageRecvedHandler = recvDecreaseMemoryQuotaResponse;
context->MessageSendReadyHandler = NULL;
context->MessageSentHandler = sentDecreaseMemoryQuota;
context->MessageErrorHandler = sentDecreaseMemoryQuotaError;
context->MessageCleanUpHandler = sentDecreaseMemoryQuotaCleanup;
res = registerAsyncConnectionFileDesc(NULL,
seghostname,
rm_segment_port,
ASYNCCOMM_READBYTES | ASYNCCOMM_WRITEBYTES,
&AsyncCommBufferHandlersMessage,
context,
&commbuffer);
if ( res != FUNC_RETURN_OK )
{
elog(LOG, "Resource manager failed to set connection to segment host %s "
"on port %d to decrease memory quota.",
seghostname,
rm_segment_port);
processContainersAfterDecreaseMemoryQuota(ctns, false);
freeGRMContainerSet(ctns);
rm_pfree(AsyncCommContext, context);
return res;
}
else
{
elog(DEBUG3, "Resource manager succeeded set connection to segment host %s "
"on port %d to decrease memory quota.",
seghostname,
rm_segment_port);
}
buildMessageToCommBuffer(commbuffer,
tosend.Buffer,
tosend.Cursor+1,
REQUEST_RM_DECREASE_MEMORY_QUOTA,
0,
0);
destroySelfMaintainBuffer(&tosend);
context->AsyncBuffer = commbuffer;
InitHandler_Message(commbuffer);
return FUNC_RETURN_OK;
}
int callSyncRPCDomain(const char *sockfile,
const char *sendbuff,
int sendbuffsize,
uint16_t sendmsgid,
uint16_t exprecvmsgid,
SelfMaintainBuffer recvsmb)
{
static char dfilename[1024];
int fd = -1;
int res = FUNC_RETURN_OK;
AsyncCommBuffer newcommbuffer = NULL;
AsyncCommMessageHandlerContext context = NULL;
SyncRPCContextData userdata;
/* Connect to the server side. */
res = connectToServerDomain(sockfile, 0, &fd, 0, dfilename);
if ( res != FUNC_RETURN_OK )
{
elog(WARNING, "Fail to connect to domain socket server %s, result %d",
sockfile,
res);
goto exit;
}
initializeSyncRPContent(&userdata, recvsmb, exprecvmsgid);
context = createMessageHandlerContext(&userdata);
res = registerFileDesc(fd,
dfilename,
ASYNCCOMM_READBYTES | ASYNCCOMM_WRITEBYTES,
&AsyncCommBufferHandlersMessage,
context,
&newcommbuffer);
if ( res != FUNC_RETURN_OK )
{
rm_pfree(AsyncCommContext, context);
elog(WARNING, "Fail to register FD for synchronous communication. %d", res);
goto exit;
}
buildMessageToCommBuffer(newcommbuffer,
sendbuff,
sendbuffsize,
sendmsgid,
0,
0);
context->AsyncBuffer = newcommbuffer;
InitHandler_Message(newcommbuffer);
/* Wait for the complete of the communication. */
while( true )
{
processAllCommFileDescs();
if ( userdata.CommStatus == SYNC_RPC_COMM_IDLE )
{
break;
}
else if ( QueryCancelPending )
{
/*
* We find that this QD wants to cancel the query, we don't need
* to continue the communication.
*/
res = TRANSCANCEL_INPROGRESS;
break;
}
}
res = res == TRANSCANCEL_INPROGRESS ? res : userdata.Result;
/* Close and cleanup */
closeAndRemoveAllRegisteredFileDesc();
if (res != FUNC_RETURN_OK)
{
elog(WARNING, "Sync RPC framework (domain) finds exception raised.");
}
return res;
exit:
closeConnectionDomain(&fd, dfilename);
return res;
}
int callSyncRPCRemote(const char *hostname,
uint16_t port,
const char *sendbuff,
int sendbuffsize,
uint16_t sendmsgid,
uint16_t exprecvmsgid,
SelfMaintainBuffer recvsmb,
char *errorbuf,
int errorbufsize)
{
int fd = -1;
int res = FUNC_RETURN_OK;
AsyncCommBuffer newcommbuffer = NULL;
AsyncCommMessageHandlerContext context = NULL;
SyncRPCContextData userdata;
/* Connect to the server side. */
res = connectToServerRemote(hostname, port, &fd);
if ( res != FUNC_RETURN_OK )
{
snprintf(errorbuf, errorbufsize,
"failed to connect to remote socket server %s:%d",
hostname,
port);
elog(WARNING, "%s", errorbuf);
goto exit;
}
initializeSyncRPContent(&userdata, recvsmb, exprecvmsgid);
context = createMessageHandlerContext(&userdata);
res = registerFileDesc(fd,
NULL,
ASYNCCOMM_READBYTES | ASYNCCOMM_WRITEBYTES,
&AsyncCommBufferHandlersMessage,
context,
&newcommbuffer);
if ( res != FUNC_RETURN_OK )
{
rm_pfree(AsyncCommContext, context);
snprintf(errorbuf, errorbufsize,
"failed to register socket connection fd %d connected to %s:%d "
"for resource rpc communication",
fd,
hostname, port);
elog(WARNING, "%s", errorbuf);
goto exit;
}
buildMessageToCommBuffer(newcommbuffer,
sendbuff,
sendbuffsize,
sendmsgid,
0,
0);
context->AsyncBuffer = newcommbuffer;
InitHandler_Message(newcommbuffer);
/* Wait for the complete of the communication. */
while( true )
{
processAllCommFileDescs();
if ( userdata.CommStatus == SYNC_RPC_COMM_IDLE )
{
break;
}
else if ( QueryCancelPending )
{
/*
* We find that this QD wants to cancel the query, we don't need
* to continue the communication.
*/
res = TRANSCANCEL_INPROGRESS;
break;
}
}
res = res == TRANSCANCEL_INPROGRESS ? res : userdata.Result;
/* Close and cleanup */
closeAndRemoveAllRegisteredFileDesc();
if (res != FUNC_RETURN_OK)
{
elog(WARNING, "Sync RPC framework (inet) finds exception raised.");
switch(res)
{
case COMM2RM_CLIENT_FAIL_SEND:
snprintf(errorbuf, errorbufsize, "failed to send content");
break;
case COMM2RM_CLIENT_FAIL_RECV:
snprintf(errorbuf, errorbufsize, "failed to receive content");
break;
case REQUESTHANDLER_WRONGMESSAGE:
snprintf(errorbuf, errorbufsize, "wrong message was received");
break;
default:
Assert(false);
}
}
return res;
exit:
elog(LOG, "Close fd %d at once", fd);
closeConnectionRemote(&fd);
return res;
}
/******************************************************************************
* I aM Alive.
*
* Request:
* |<----------- 64 bits (8 bytes) ----------->|
* +----------+--------------------------------+
* | TDC | BDC | Reserved |
* +----------+----------+---------------------+
* | |
* | Machine ID info |
* | |
* +-------------------------------------------+ _____ 64bit aligned
*
* Response:
* |<----------- 64 bits (8 bytes) ----------->|
* +---------------------+---------------------+
* | heartbeat result | Reserved |
* +---------------------+---------------------+ _____ 64bit aligned
*
******************************************************************************/
int sendIMAlive(int *errorcode,
char *errorbuf,
int errorbufsize)
{
int res = FUNC_RETURN_OK;
AsyncCommBuffer newcommbuffer = NULL;
Assert( DRMGlobalInstance->LocalHostStat != NULL );
/* Build request. */
SelfMaintainBufferData tosend;
initializeSelfMaintainBuffer(&tosend, PCONTEXT);
RPCRequestHeadIMAliveData requesthead;
requesthead.TmpDirCount = getDQueueLength(&DRMGlobalInstance->LocalHostTempDirectories);
requesthead.TmpDirBrokenCount = DRMGlobalInstance->LocalHostStat->FailedTmpDirNum;
requesthead.Reserved = 0;
appendSMBVar(&tosend, requesthead);
appendSelfMaintainBuffer(&tosend,
(char *)(DRMGlobalInstance->LocalHostStat),
offsetof(SegStatData, Info) +
DRMGlobalInstance->LocalHostStat->Info.Size);
/* Set content to send and add to AsyncComm framework. */
AsyncCommMessageHandlerContext context =
rm_palloc0(AsyncCommContext,
sizeof(AsyncCommMessageHandlerContextData));
context->inMessage = false;
context->UserData = NULL;
context->MessageRecvReadyHandler = NULL;
context->MessageRecvedHandler = receivedIMAliveResponse;
context->MessageSendReadyHandler = NULL;
context->MessageSentHandler = sentIMAlive;
context->MessageErrorHandler = sentIMAliveError;
context->MessageCleanUpHandler = sentIMAliveCleanUp;
/* Connect to HAWQ RM server */
res = registerAsyncConnectionFileDesc(NULL,
DRMGlobalInstance->SendToStandby?
standby_addr_host:
master_addr_host,
rm_master_port,
ASYNCCOMM_READBYTES | ASYNCCOMM_WRITEBYTES,
&AsyncCommBufferHandlersMessage,
context,
&newcommbuffer);
if ( res != FUNC_RETURN_OK )
{
rm_pfree(AsyncCommContext, context);
elog(WARNING, "Fail to register asynchronous connection for sending "
"IMAlive message. %d", res);
return res;
}
buildMessageToCommBuffer(newcommbuffer,
tosend.Buffer,
tosend.Cursor + 1,
REQUEST_RM_IMALIVE,
0,
0);
destroySelfMaintainBuffer(&tosend);
context->AsyncBuffer = newcommbuffer;
InitHandler_Message(newcommbuffer);
return FUNC_RETURN_OK;
}